diff --git a/.github/workflows/api-test-lint-deploy.yaml b/.github/workflows/api-test-lint-deploy.yaml index 899bffa7c4e..f3d0532e8a8 100644 --- a/.github/workflows/api-test-lint-deploy.yaml +++ b/.github/workflows/api-test-lint-deploy.yaml @@ -129,6 +129,8 @@ jobs: if: github.event_name == 'push' steps: - uses: 'actions/checkout@v3' + with: + fetch-depth: 0 - uses: 'actions/setup-node@v3' with: node-version: '16' diff --git a/.github/workflows/docs-build.yaml b/.github/workflows/docs-build.yaml index a5eda6b7ce8..9ce5871b4d5 100644 --- a/.github/workflows/docs-build.yaml +++ b/.github/workflows/docs-build.yaml @@ -41,6 +41,8 @@ jobs: runs-on: 'ubuntu-22.04' steps: - uses: 'actions/checkout@v3' + with: + fetch-depth: 0 - uses: 'actions/setup-node@v3' with: node-version: '16' diff --git a/.github/workflows/hardware-lint-test.yaml b/.github/workflows/hardware-lint-test.yaml index f568000a303..552fd6b21d7 100644 --- a/.github/workflows/hardware-lint-test.yaml +++ b/.github/workflows/hardware-lint-test.yaml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout opentrons repo uses: 'actions/checkout@v3' - + fetch-depth: 0 - name: Setup Node uses: 'actions/setup-node@v3' with: diff --git a/.github/workflows/hardware-testing-protocols.yaml b/.github/workflows/hardware-testing-protocols.yaml index 3b4fc153094..69275e8416b 100644 --- a/.github/workflows/hardware-testing-protocols.yaml +++ b/.github/workflows/hardware-testing-protocols.yaml @@ -39,6 +39,7 @@ jobs: steps: - name: Checkout opentrons repo uses: 'actions/checkout@v3' + fetch-depth: 0 - name: Setup Node uses: 'actions/setup-node@v3' diff --git a/.github/workflows/shared-data-test-lint-deploy.yaml b/.github/workflows/shared-data-test-lint-deploy.yaml index dfb678a4477..5327675f3cb 100644 --- a/.github/workflows/shared-data-test-lint-deploy.yaml +++ b/.github/workflows/shared-data-test-lint-deploy.yaml @@ -144,6 +144,8 @@ jobs: if: github.event_name == 'push' steps: - uses: 'actions/checkout@v3' + with: + fetch-depth: 0 - uses: 'actions/setup-node@v3' with: node-version: '16' diff --git a/api/MANIFEST.in b/api/MANIFEST.in index 55b69639860..66a25cf55b5 100755 --- a/api/MANIFEST.in +++ b/api/MANIFEST.in @@ -1,4 +1,3 @@ -include src/opentrons/package.json graft src/opentrons/config graft src/opentrons/resources exclude pyproject.toml diff --git a/api/Makefile b/api/Makefile index b3f05e07560..480de1e01e1 100755 --- a/api/Makefile +++ b/api/Makefile @@ -17,14 +17,17 @@ sphinx_build := $(pipenv) run sphinx-build -W --keep-going # github.com/Opentrons/opentrons/issues/6135 sphinx_build_allow_warnings := $(pipenv) run sphinx-build +ot_project := $(OPENTRONS_PROJECT) +project_rs_default = $(if $(ot_project),$(ot_project),robot-stack) +project_ot3_default = $(if $(ot_project),$(ot_project),ot3) -# Find the version of the wheel from package.json using a helper script. We +# Find the version of the wheel from git using a helper script. We # use python here so we can use the same version normalization that will be # used to create the wheel. -wheel_file = dist/$(call python_get_wheelname,api,opentrons,$(BUILD_NUMBER)) +wheel_file = dist/$(call python_get_wheelname,api,$(project_rs_default),opentrons,$(BUILD_NUMBER)) -# Find the version of the sdist file from package.json using a helper script. -sdist_file = dist/$(call python_get_sdistname,api,opentrons) +# Find the version of the sdist file from git using a helper script. +sdist_file = dist/$(call python_get_sdistname,api,$(project_ot3_default),opentrons) # These variables are for simulating python protocols sim_log_level ?= info @@ -51,18 +54,12 @@ ssh_opts ?= $(default_ssh_opts) twine_auth_args := --username $(pypi_username) --password $(pypi_password) twine_repository_url ?= $(pypi_test_upload_url) -# Source discovery -# For the python sources -ot_py_sources := $(filter %.py,$(shell $(SHX) find src/opentrons/)) -# And the out of tree shared data -ot_shared_data_sources := $(filter %.json,$(shell $(SHX) find ../shared-data/)) -# And the arbitrary stuff in resources -ot_resources := $(filter %,$(shell $(SHX) find src/opentrons/resources)) -ot_sources := $(ot_py_sources) $(ot_shared_data_sources) $(ot_resources) - # Defined separately than the clean target so the wheel file doesn’t have to # depend on a PHONY target -clean_cmd = $(SHX) rm -rf build dist .coverage coverage.xml '*.egg-info' '**/__pycache__' '**/*.pyc' 'src/**/.mypy_cache' +clean_cmd = $(SHX) rm -rf build .coverage coverage.xml '*.egg-info' '**/__pycache__' '**/*.pyc' 'src/**/.mypy_cache' +clean_wheel_cmd = $(clean_cmd) dist/*.whl +clean_sdist_cmd = $(clean_cmd) dist/*.tar.gz +clean_all_cmd = $(clean_cmd) dist plot_type ?= @@ -82,28 +79,29 @@ setup-ot2: .PHONY: clean clean: docs-clean - $(clean_cmd) + $(clean_all_cmd) .PHONY: teardown teardown: $(pipenv) --rm -$(wheel_file): setup.py $(ot_sources) - $(clean_cmd) +.PHONY: wheel +wheel: export OPENTRONS_PROJECT=$(project_rs_default) +wheel: + $(clean_wheel_cmd) $(python) setup.py $(wheel_opts) bdist_wheel $(SHX) rm -rf build $(SHX) ls dist -wheel: $(wheel_file) -$(sdist_file): setup.py $(ot_sources) - $(clean_cmd) +.PHONY: sdist +sdist: export OPENTRONS_PROJECT=$(project_ot3_default) +sdist: + $(clean_sdist_cmd) $(python) setup.py sdist $(SHX) rm -rf build $(SHX) ls dist -sdist: $(sdist_file) - .PHONY: test test: $(pytest) $(tests) $(test_opts) diff --git a/api/buildroot.mk b/api/buildroot.mk index 89904a7d7f4..1d377169515 100644 --- a/api/buildroot.mk +++ b/api/buildroot.mk @@ -4,13 +4,12 @@ # ################################################################################ -# Get a key from package.json (like version) -define get_pkg_json_key - $(shell python -c "import json; print(json.load(open('$(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/api/src/opentrons/package.json'))[\"$(1)\"])") +define OTAPI_CALL_PBU + $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py api robot-stack $(1)) endef -PYTHON_OPENTRONS_API_VERSION = $(call get_pkg_json_key,version) -PYTHON_OPENTRONS_API_LICENSE = $(call get_pkg_json_key,license) +PYTHON_OPENTRONS_API_VERSION = $(call OTAPI_CALL_PBU,get_version) +PYTHON_OPENTRONS_API_LICENSE = Apache-2 PYTHON_OPENTRONS_API_LICENSE_FILES = $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/LICENSE PYTHON_OPENTRONS_API_SETUP_TYPE = setuptools PYTHON_OPENTRONS_API_SITE_METHOD = local @@ -18,12 +17,8 @@ PYTHON_OPENTRONS_API_SITE = $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH) PYTHON_OPENTRONS_API_SUBDIR = api PYTHON_OPENTRONS_API_POST_INSTALL_TARGET_HOOKS = PYTHON_OPENTRONS_API_INSTALL_VERSION PYTHON_OPENTRONS_API_INSTALL_RELEASE_NOTES -define OTAPI_DUMP_BR_VERSION - $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py api dump_br_version) -endef - define PYTHON_OPENTRONS_API_INSTALL_VERSION - echo '$(call OTAPI_DUMP_BR_VERSION)' > $(BINARIES_DIR)/opentrons-api-version.json + echo '$(call OTAPI_CALL_PBU,dump_br_version)' > $(BINARIES_DIR)/opentrons-api-version.json endef ot_api_name := python-opentrons-api diff --git a/api/docs/hardware/conf.py b/api/docs/hardware/conf.py index abdeba95583..bbaadf2cf67 100644 --- a/api/docs/hardware/conf.py +++ b/api/docs/hardware/conf.py @@ -21,9 +21,6 @@ import sys import json import pkgutil -# This load means sphinx docs builds must be run in a virtualenv with the -# newly-built module -_package_json = json.loads(pkgutil.get_data('opentrons', 'package.json')) sys.path.insert(0, os.path.abspath('../..')) sys.path.insert(0, os.path.abspath('../sphinxext')) @@ -66,16 +63,20 @@ # General information about the project. project = 'OT-2 Hardware Control API' copyright = '2020, Opentrons' -author = _package_json['author']['name'] +author = 'Opentrons Labworks' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. -# +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts')) +import python_build_utils +sys.path = sys.path[:-1] +_vers = python_build_utils.get_version('api', 'robot-stack') + # The short X.Y version. -version = '.'.join(_package_json['version'].split('.')[:2]) +version = '.'.join(_vers.split('.')[:2]) # The full version, including alpha/beta/rc tags. -release = _package_json['version'] +release = _vers # setup the code block substitution extension to auto-update apiLevel extensions += ['sphinx-prompt', 'sphinx_substitution_extensions'] @@ -320,7 +321,7 @@ latex_documents = [ (master_doc, 'OT2HardwareControl.tex', 'OT-2 Hardware Control API Documentation', - _package_json['author']['name'], 'howto'), + 'Opentrons Labworks', 'howto'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/api/docs/v1/conf.py b/api/docs/v1/conf.py index ad4cf6e7a44..6f0f4fc6f20 100644 --- a/api/docs/v1/conf.py +++ b/api/docs/v1/conf.py @@ -21,9 +21,6 @@ import sys import json import pkgutil -# This load means sphinx docs builds must be run in a virtualenv with the -# newly-built module -_package_json = json.loads(pkgutil.get_data('opentrons', 'package.json')) sys.path.insert(0, os.path.abspath('./api_cache')) sys.path.insert(0, os.path.abspath('../..')) @@ -67,16 +64,21 @@ # General information about the project. project = 'OT-2 API V1' copyright = '2019, Opentrons' -author = _package_json['author']['name'] +author = "Opentrons Labworks" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. -# + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts')) +import python_build_utils +sys.path = sys.path[:-1] +_vers = python_build_utils.get_version('api', 'robot-stack') + # The short X.Y version. -version = '.'.join(_package_json['version'].split('.')[:2]) +version = '.'.join(_vers.split('.')[:2]) # The full version, including alpha/beta/rc tags. -release = _package_json['version'] +release = _vers # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -311,7 +313,7 @@ latex_documents = [ (master_doc, 'OpentronsPythonAPIV1.tex', 'OT-2 Python API V1 Documentation', - _package_json['author']['name'], 'howto'), + 'Opentrons Labworks', 'howto'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/api/docs/v2/conf.py b/api/docs/v2/conf.py index 0163427f337..52262acb5e2 100644 --- a/api/docs/v2/conf.py +++ b/api/docs/v2/conf.py @@ -21,9 +21,6 @@ import sys import json import pkgutil -# This load means sphinx docs builds must be run in a virtualenv with the -# newly-built module -_package_json = json.loads(pkgutil.get_data('opentrons', 'package.json')) sys.path.insert(0, os.path.abspath('../..')) sys.path.insert(0, os.path.abspath('../sphinxext')) @@ -74,7 +71,7 @@ # General information about the project. project = 'OT-2 API V2' copyright = '2010, Opentrons' -author = _package_json['author']['name'] +author = 'Opentrons Labworks' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -82,11 +79,17 @@ # # todo(mm, 2021-09-30): Depending on where these show up, would it be more correct # to use the latest-supported *apiLevel* instead of the *Python package version*? -# + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts')) +import python_build_utils +sys.path = sys.path[:-1] +_vers = python_build_utils.get_version('api', 'robot-stack') + + # The short X.Y version. -version = '.'.join(_package_json['version'].split('.')[:2]) +version = '.'.join(_vers.split('.')[:2]) # The full version, including alpha/beta/rc tags. -release = _package_json['version'] +release = _vers # setup the code block substitution extension to auto-update apiLevel extensions += ['sphinx-prompt', 'sphinx_substitution_extensions'] @@ -333,7 +336,7 @@ latex_documents = [ (master_doc, 'OpentronsPythonAPIV2.tex', 'Opentrons Python API V2 Documentation', - _package_json['author']['name'], 'howto'), + 'Opentrons Labworks', 'howto'), ] # The name of an image file (relative to this directory) to place at the top of diff --git a/api/setup.py b/api/setup.py index 0bcd3e9b11d..c78161b5614 100755 --- a/api/setup.py +++ b/api/setup.py @@ -22,11 +22,12 @@ def get_version(): buildno = os.getenv("BUILD_NUMBER") + project = os.getenv("OPENTRONS_PROJECT", "robot-stack") if buildno: normalize_opts = {"extra_tag": buildno} else: normalize_opts = {} - return normalize_version("api", **normalize_opts) + return normalize_version("api", project, **normalize_opts) VERSION = get_version() @@ -66,6 +67,7 @@ def get_version(): "pyserial==3.5", "typing-extensions>=4.0.0,<5", "click>=8.0.0,<9", + 'importlib-metadata >= 1.0 ; python_version < "3.8"', ] @@ -97,7 +99,7 @@ def read(*parts): install_requires=INSTALL_REQUIRES, include_package_data=True, package_dir={"": "src"}, - package_data={"opentrons": ["py.typed", "package.json"]}, + package_data={"opentrons": ["py.typed"]}, entry_points={ "console_scripts": [ "opentrons_simulate = opentrons.simulate:main", diff --git a/api/src/opentrons/__init__.py b/api/src/opentrons/__init__.py index 48d38ae5360..499c51c1b39 100755 --- a/api/src/opentrons/__init__.py +++ b/api/src/opentrons/__init__.py @@ -1,6 +1,5 @@ -import json import os -import sys + from pathlib import Path import logging import asyncio @@ -25,28 +24,17 @@ from opentrons.protocols.types import ApiDeprecationError from opentrons.protocols.api_support.types import APIVersion -version = sys.version_info[0:2] -if version < (3, 7): - raise RuntimeError( - "opentrons requires Python 3.7 or above, this is {0}.{1}".format( - version[0], version[1] - ) - ) - HERE = os.path.abspath(os.path.dirname(__file__)) +from ._version import version # noqa: E402 -try: - with open(os.path.join(HERE, "package.json")) as pkg: - package_json = json.load(pkg) - __version__ = package_json.get("version") -except (FileNotFoundError, OSError): - __version__ = "unknown" +__version__ = version from opentrons import config # noqa: E402 LEGACY_MODULES = ["robot", "reset", "instruments", "containers", "labware", "modules"] -__all__ = ["version", "HERE", "config"] + +__all__ = ["version", "__version__", "HERE", "config"] def __getattr__(attrname: str) -> None: @@ -155,7 +143,7 @@ async def initialize() -> ThreadManagedHardware: robot_conf = robot_configs.load() logging_config.log_init(robot_conf.log_level) - log.info(f"API server version: {__version__}") + log.info(f"API server version: {version}") log.info(f"Robot Name: {name()}") hardware = await _create_thread_manager() diff --git a/api/src/opentrons/_version.py b/api/src/opentrons/_version.py new file mode 100644 index 00000000000..dc3208c24e3 --- /dev/null +++ b/api/src/opentrons/_version.py @@ -0,0 +1,27 @@ +import sys +import logging + +_pyversion = sys.version_info[0:2] +if _pyversion < (3, 7): + raise RuntimeError( + "opentrons requires Python 3.7 or above, this is {0}.{1}".format( + _pyversion[0], _pyversion[1] + ) + ) + + +if _pyversion >= (3, 8): + from importlib import metadata +else: + import importlib_metadata as metadata # type: ignore[no-redef] + +try: + version: str = metadata.version("opentrons") # type: ignore[attr-defined] +except Exception as e: + logging.warning( + "Could not determine version for opentrons, may be dev install, using 0.0.0-dev" + ) + logging.debug(e) + version = "0.0.0-dev" + +__all__ = ["version"] diff --git a/api/src/opentrons/package.json b/api/src/opentrons/package.json deleted file mode 100644 index 40ef35bc743..00000000000 --- a/api/src/opentrons/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@opentrons/api-server", - "version": "6.2.0", - "description": "Opentrons API server application", - "repository": { - "type": "git", - "url": "https://github.com/Opentrons/opentrons.git" - }, - "author": { - "name": "Opentrons Labworks", - "email": "engineering@opentrons.com" - }, - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/Opentrons/opentrons/issues" - }, - "homepage": "https://github.com/Opentrons/opentrons" -} diff --git a/g-code-testing/setup.py b/g-code-testing/setup.py index 55fe4ec72dd..debfc643fa2 100644 --- a/g-code-testing/setup.py +++ b/g-code-testing/setup.py @@ -14,18 +14,19 @@ fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) HERE = os.path.abspath(os.path.dirname(__file__)) -sys.path.append(os.path.join(HERE, "..", "..", "..", "scripts")) +sys.path.append(os.path.join(HERE, "..", "scripts")) from python_build_utils import normalize_version # noqa: E402 def get_version(): buildno = os.getenv("BUILD_NUMBER") + project = os.getenv("OPENTRONS_PROJECT", "robot-stack") if buildno: normalize_opts = {"extra_tag": buildno} else: normalize_opts = {} - return normalize_version("g-code-testing", **normalize_opts) + return normalize_version("g-code-testing", project, **normalize_opts) VERSION = get_version() @@ -64,22 +65,23 @@ def read(*parts): return f.read() -setup( - python_requires=">=3.7", - name=DISTNAME, - description=DESCRIPTION, - license=LICENSE, - url=URL, - version=VERSION, - author=AUTHOR, - author_email=EMAIL, - maintainer=AUTHOR, - maintainer_email=EMAIL, - keywords=KEYWORDS, - long_description=__doc__, - packages=PACKAGES, - zip_safe=False, - classifiers=CLASSIFIERS, - install_requires=INSTALL_REQUIRES, - include_package_data=True, -) +if __name__ == "__main__": + setup( + python_requires=">=3.7", + name=DISTNAME, + description=DESCRIPTION, + license=LICENSE, + url=URL, + version=VERSION, + author=AUTHOR, + author_email=EMAIL, + maintainer=AUTHOR, + maintainer_email=EMAIL, + keywords=KEYWORDS, + long_description=__doc__, + packages=PACKAGES, + zip_safe=False, + classifiers=CLASSIFIERS, + install_requires=INSTALL_REQUIRES, + include_package_data=True, + ) diff --git a/hardware/Makefile b/hardware/Makefile index 28483a67863..cb31e9f490c 100755 --- a/hardware/Makefile +++ b/hardware/Makefile @@ -5,16 +5,20 @@ include ../scripts/python.mk SHX := npx shx -# Find the version of the wheel from package.json using a helper script. We +ot_project := $(OPENTRONS_PROJECT) +project_rs_default = $(if $(ot_project),$(ot_project),robot-stack) +project_ot3_default = $(if $(ot_project),$(ot_project),ot3) + +# Find the version of the wheel from git using a helper script. We # use python here so we can use the same version normalization that will be # used to create the wheel. -wheel_file = dist/$(call python_get_wheelname,hardware,opentrons_hardware,$(BUILD_NUMBER)) +wheel_file = dist/$(call python_get_wheelname,hardware,$(project_rs_default),opentrons_hardware,$(BUILD_NUMBER)) -# Find the version of the sdist from package.json using a helper script. We +# Find the version of the sdist from git using a helper script. We # use python here so we can use the same version normalization that will be # used to create the sdist. -sdist_file = dist/$(call python_get_sdistname,hardware,opentrons_hardware) +sdist_file = dist/$(call python_get_sdistname,hardware,$(project_ot3_default),opentrons_hardware) # These variables can be overriden when make is invoked to customize the @@ -57,12 +61,13 @@ setup: .PHONY: teardown teardown: - $(pipenv) --rm + -$(pipenv) --rm .PHONY: clean clean: - $(clean_cmd) + -$(clean_cmd) +dist/opentrons_hardware-%-py2.py3-none-any.whl: export OPENTRONS_PROJECT=$(project_rs_default) dist/opentrons_hardware-%-py2.py3-none-any.whl: setup.py $(ot_sources) $(python) setup.py $(wheel_opts) bdist_wheel $(SHX) rm -rf build @@ -70,6 +75,7 @@ dist/opentrons_hardware-%-py2.py3-none-any.whl: setup.py $(ot_sources) wheel: $(wheel_file) +$(sdist_file): export OPENTRONS_PROJECT=$(project_ot3_default) $(sdist_file): setup.py $(ot_sources) $(python) setup.py sdist $(SHX) rm -rf build diff --git a/hardware/opentrons_hardware/package.json b/hardware/opentrons_hardware/package.json deleted file mode 100644 index ceac88ffb17..00000000000 --- a/hardware/opentrons_hardware/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@opentrons/hardware", - "version": "4.5.0", - "description": "Opentrons API server application", - "repository": { - "type": "git", - "url": "https://github.com/Opentrons/opentrons.git" - }, - "author": { - "name": "Opentrons Labworks", - "email": "engineering@opentrons.com" - }, - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/Opentrons/opentrons/issues" - }, - "homepage": "https://github.com/Opentrons/opentrons" -} diff --git a/hardware/setup.py b/hardware/setup.py index 50c5c23e796..f285eaf3128 100644 --- a/hardware/setup.py +++ b/hardware/setup.py @@ -17,11 +17,12 @@ def get_version() -> str: """Get the version.""" buildno = os.getenv("BUILD_NUMBER") + project = os.getenv("OPENTRONS_PROJECT", "robot-stack") if buildno: normalize_opts = {"extra_tag": buildno} else: normalize_opts = {} - return normalize_version("hardware", **normalize_opts) + return normalize_version("hardware", project, **normalize_opts) VERSION = get_version() diff --git a/notify-server/Makefile b/notify-server/Makefile index 754478e3670..4c657b36322 100755 --- a/notify-server/Makefile +++ b/notify-server/Makefile @@ -15,15 +15,20 @@ SHX := npx shx # Path of source package SRC_PATH = notify_server -# Find the version of the wheel from package.json using a helper script. We +ot_project := $(OPENTRONS_PROJECT) +project_rs_default = $(if $(ot_project),$(ot_project),robot-stack) +project_ot3_default = $(if $(ot_project),$(ot_project),ot3) + + +# Find the version of the wheel from git using a helper script. We # use python here so we can use the same version normalization that will be # used to create the wheel. -wheel_file = dist/$(call python_get_wheelname,notify-server,notify_server,$(BUILD_NUMBER)) +wheel_file = dist/$(call python_get_wheelname,notify-server,$(project_rs_default),notify_server,$(BUILD_NUMBER)) -# Find the version of the sdist from package.json using a helper script. We +# Find the version of the sdist from git using a helper script. We # use python here so we can use the same version normalization that will be # used to create the sdist. -sdist_file = dist/$(call python_get_sdistname,notify-server,notify_server) +sdist_file = dist/$(call python_get_sdistname,notify-server,$(project_ot3_default),notify_server) # These variables can be overriden when make is invoked to customize the @@ -72,6 +77,7 @@ clean: teardown: $(pipenv) --rm +$(wheel_file): export OPENTRONS_PROJECT=$(project_rs_default) $(wheel_file): setup.py $(ot_sources) $(clean_cmd) $(python) setup.py $(wheel_opts) bdist_wheel --universal @@ -81,6 +87,7 @@ $(wheel_file): setup.py $(ot_sources) .PHONY: wheel wheel: $(wheel_file) +$(sdist_file): export OPENTRONS_PROJECT=$(project_ot3_default) $(sdist_file): setup.py $(ot_sources) $(clean_cmd) $(python) setup.py sdist diff --git a/notify-server/buildroot.mk b/notify-server/buildroot.mk index 25992abeb15..3d5712985a9 100644 --- a/notify-server/buildroot.mk +++ b/notify-server/buildroot.mk @@ -4,13 +4,13 @@ # ################################################################################ -# Get a key from package.json (like version) -define PYTHON_OPENTRONS_NOTIFY_SERVER_get_pkg_json_key - $(shell python -c "import json; print(json.load(open('$(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/notify-server/notify_server/package.json'))[\"$(1)\"])") + +define OTNOTIFYSERVER_CALL_PBU + $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py notify-server robot-stack $(1)) endef -PYTHON_OPENTRONS_NOTIFY_SERVER_VERSION = $(call PYTHON_OPENTRONS_NOTIFY_SERVER_get_pkg_json_key,version) -PYTHON_OPENTRONS_NOTIFY_SERVER_LICENSE = $(call PYTHON_OPENTRONS_NOTIFY_SERVER_get_pkg_json_key,license) +PYTHON_OPENTRONS_NOTIFY_SERVER_VERSION = $(call OTNOTIFYSERVER_CALL_PBU,get_version) +PYTHON_OPENTRONS_NOTIFY_SERVER_LICENSE = Apache-2 PYTHON_OPENTRONS_NOTIFY_SERVER_LICENSE_FILES = $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/LICENSE PYTHON_OPENTRONS_NOTIFY_SERVER_SETUP_TYPE = setuptools PYTHON_OPENTRONS_NOTIFY_SERVER_SITE_METHOD = local @@ -19,12 +19,8 @@ PYTHON_OPENTRONS_NOTIFY_SERVER_SUBDIR = notify-server PYTHON_OPENTRONS_NOTIFY_SERVER_POST_INSTALL_TARGET_HOOKS = PYTHON_OPENTRONS_NOTIFY_SERVER_INSTALL_VERSION PYTHON_OPENTRONS_NOTIFY_SERVER_SERVICE_FILE_NAME=opentrons-notify-server.service -define OTNOTIFYSERVER_DUMP_BR_VERSION - $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py notify-server dump_br_version) -endef - define PYTHON_OPENTRONS_NOTIFY_SERVER_INSTALL_VERSION - echo '$(call OTNOTIFYSERVER_DUMP_BR_VERSION)' > $(BINARIES_DIR)/opentrons-notify-server-version.json + echo '$(call OTNOTIFYSERVER_CALL_PBU,dump_br_version)' > $(BINARIES_DIR)/opentrons-notify-server-version.json endef ot_notify_server_name := python-opentrons-notify-server diff --git a/notify-server/notify_server/__init__.py b/notify-server/notify_server/__init__.py index 155f27b1761..30684af468a 100644 --- a/notify-server/notify_server/__init__.py +++ b/notify-server/notify_server/__init__.py @@ -1,3 +1 @@ """Notify Server Package.""" - -__version__ = "0.1.0" diff --git a/notify-server/notify_server/package.json b/notify-server/notify_server/package.json deleted file mode 100644 index 5ee49a6853d..00000000000 --- a/notify-server/notify_server/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@opentrons/notify-server", - "version": "6.2.0", - "description": "Notification pub/sub server for Opentrons robots", - "repository": { - "type": "git", - "url": "https://github.com/Opentrons/opentrons.git" - }, - "author": { - "name": "Opentrons Labworks", - "email": "engineering@opentrons.com" - }, - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/Opentrons/opentrons/issues" - }, - "homepage": "https://github.com/Opentrons/opentrons" -} diff --git a/notify-server/setup.py b/notify-server/setup.py index 9441ca2fd3d..ab5854b32b9 100644 --- a/notify-server/setup.py +++ b/notify-server/setup.py @@ -14,11 +14,12 @@ def get_version(): buildno = os.getenv("BUILD_NUMBER") + project = os.getenv("OPENTRONS_PROJECT", "robot-stack") if buildno: normalize_opts = {"extra_tag": buildno} else: normalize_opts = {} - return normalize_version("notify-server", **normalize_opts) + return normalize_version("notify-server", project, **normalize_opts) VERSION = get_version() diff --git a/notify-server/tests/unit/test_notify_server.py b/notify-server/tests/unit/test_notify_server.py deleted file mode 100644 index 3be8f64d8d0..00000000000 --- a/notify-server/tests/unit/test_notify_server.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Test entry.""" -from notify_server import __version__ - - -def test_version() -> None: - """Test version.""" - assert __version__ == "0.1.0" diff --git a/package.json b/package.json index 48975904b06..f96285386c7 100755 --- a/package.json +++ b/package.json @@ -2,7 +2,6 @@ "name": "opentrons", "private": true, "workspaces": [ - "api/src/opentrons", "app", "app-shell", "components", @@ -11,13 +10,9 @@ "labware-library", "protocol-designer", "protocol-library-kludge", - "robot-server/robot_server", "shared-data", "step-generation", - "update-server/otupdate", "webpack-config", - "notify-server/notify_server", - "hardware/hardware", "api-client", "react-api-client" ], diff --git a/robot-server/MANIFEST.in b/robot-server/MANIFEST.in deleted file mode 100755 index ae8a37970b7..00000000000 --- a/robot-server/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include robot_server/package.json diff --git a/robot-server/Makefile b/robot-server/Makefile index 1602654e81c..46bca8e8cec 100755 --- a/robot-server/Makefile +++ b/robot-server/Makefile @@ -8,15 +8,21 @@ SHX := npx shx # Path of source package SRC_PATH = robot_server -# Find the version of the wheel from package.json using a helper script. We +# Project to get the version for +ot_project := $(OPENTRONS_PROJECT) +project_rs_default = $(if $(ot_project),$(ot_project),robot-stack) +project_ot3_default = $(if $(ot_project),$(ot_project),ot3) + + +# Find the version of the wheel from git using a helper script. We # use python here so we can use the same version normalization that will be # used to create the wheel. -wheel_file = dist/$(call python_get_wheelname,robot-server,robot_server,$(BUILD_NUMBER)) +wheel_file = dist/$(call python_get_wheelname,robot-server,$(project_rs_default),robot_server,$(BUILD_NUMBER)) -# Find the version of the sdist from package.json using a helper script. We +# Find the version of the sdist from git using a helper script. We # use python here so we can use the same version normalization that will be # used to create the sdist. -sdist_file = dist/$(call python_get_sdistname,robot-server,robot_server) +sdist_file = dist/$(call python_get_sdistname,robot-server,$(project_ot3_default),robot_server) # These variables can be overriden when make is invoked to customize the # behavior of pytest. For instance, @@ -40,8 +46,10 @@ ot_sources := $(ot_py_sources) # Defined separately than the clean target so the wheel file doesn’t have to # depend on a PHONY target -clean_cmd = $(SHX) rm -rf build dist .coverage coverage.xml '*.egg-info' '**/__pycache__' '**/*.pyc' 'robot_server/**/.mypy_cache' - +clean_cmd = $(SHX) rm -rf build .coverage coverage.xml '*.egg-info' '**/__pycache__' '**/*.pyc' 'robot_server/**/.mypy_cache' +clean_wheel_cmd = $(clean_cmd) dist/*.whl +clean_sdist_cmd = $(clean_cmd) dist/*.tar.gz +clean_all_cmd = $(clean_cmd) dist # Uvicorn command to run the robot server in dev mode. # # "--lifespan on" ensures errors in startup and shutdown handlers won't be hidden. @@ -75,28 +83,28 @@ setup-ot2: .PHONY: clean clean: - $(clean_cmd) + $(clean_all_cmd) .PHONY: teardown teardown: $(pipenv) --rm -$(wheel_file): setup.py $(ot_sources) - $(clean_cmd) +.PHONY: wheel +wheel: export OPENTRONS_PROJECT=$(project_rs_default) +wheel: setup.py $(ot_sources) + $(clean_wheel_cmd) $(python) setup.py $(wheel_opts) bdist_wheel $(SHX) rm -rf build $(SHX) ls dist -wheel: $(wheel_file) - -$(sdist_file): setup.py $(ot_sources) - $(clean_cmd) +.PHONY: sdist +sdist: export OPENTRONS_PROJECT=$(project_ot3_default) +sdist: setup.py $(ot_sources) + $(clean_sdist_cmd) $(python) setup.py sdist $(SHX) rm -rf build $(SHX) ls dist -sdist: $(sdist_file) - .PHONY: test test: $(pytest) $(tests) $(test_opts) diff --git a/robot-server/buildroot.mk b/robot-server/buildroot.mk index f353d0efee6..e8c4a246fd8 100644 --- a/robot-server/buildroot.mk +++ b/robot-server/buildroot.mk @@ -4,13 +4,12 @@ # ################################################################################ -# Get a key from package.json (like version) -define get_pkg_json_key - $(shell python -c "import json; print(json.load(open('$(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/robot-server/robot_server/package.json'))[\"$(1)\"])") +define OTROBOTSERVER_CALL_PBU + $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py robot-server robot-stack $(1)) endef -PYTHON_OPENTRONS_ROBOT_SERVER_VERSION = $(call get_pkg_json_key,version) -PYTHON_OPENTRONS_ROBOT_SERVER_LICENSE = $(call get_pkg_json_key,license) +PYTHON_OPENTRONS_ROBOT_SERVER_VERSION = $(call OTROBOTSERVER_CALL_PBU,get_version) +PYTHON_OPENTRONS_ROBOT_SERVER_LICENSE = Apache-2 PYTHON_OPENTRONS_ROBOT_SERVER_LICENSE_FILES = $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/LICENSE PYTHON_OPENTRONS_ROBOT_SERVER_SETUP_TYPE = setuptools PYTHON_OPENTRONS_ROBOT_SERVER_SITE_METHOD = local @@ -19,12 +18,8 @@ PYTHON_OPENTRONS_ROBOT_SERVER_SUBDIR = robot-server PYTHON_OPENTRONS_ROBOT_SERVER_POST_INSTALL_TARGET_HOOKS = PYTHON_OPENTRONS_ROBOT_SERVER_INSTALL_VERSION PYTHON_OPENTRONS_ROBOT_SERVER_SERVICE_FILE_NAME=opentrons-robot-server.service -define OTROBOTSERVER_DUMP_BR_VERSION - $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py robot-server dump_br_version) -endef - define PYTHON_OPENTRONS_ROBOT_SERVER_INSTALL_VERSION - echo '$(call OTROBOTSERVER_DUMP_BR_VERSION)' > $(BINARIES_DIR)/opentrons-robot-server-version.json + echo '$(call OTROBOTSERVER_CALL_PBU,dump_br_version)' > $(BINARIES_DIR)/opentrons-robot-server-version.json endef ot_robot_server_name := python-opentrons-robot-server diff --git a/robot-server/robot_server/package.json b/robot-server/robot_server/package.json deleted file mode 100644 index a1a3ed446ad..00000000000 --- a/robot-server/robot_server/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@opentrons/robot-server", - "version": "6.2.0", - "description": "HTTP server for Opentrons robots", - "repository": { - "type": "git", - "url": "https://github.com/Opentrons/opentrons.git" - }, - "author": { - "name": "Opentrons Labworks", - "email": "engineering@opentrons.com" - }, - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/Opentrons/opentrons/issues" - }, - "homepage": "https://github.com/Opentrons/opentrons" -} diff --git a/robot-server/setup.py b/robot-server/setup.py index a46af0ae81e..faaf0abbbf2 100755 --- a/robot-server/setup.py +++ b/robot-server/setup.py @@ -21,11 +21,12 @@ def get_version(): buildno = os.getenv("BUILD_NUMBER") + project = os.getenv("OPENTRONS_PROJECT", "robot-stack") if buildno: normalize_opts = {"extra_tag": buildno} else: normalize_opts = {} - return normalize_version("robot-server", **normalize_opts) + return normalize_version("robot-server", project, **normalize_opts) VERSION = get_version() diff --git a/scripts/python.mk b/scripts/python.mk index baeb617a84a..39329d93a9d 100644 --- a/scripts/python.mk +++ b/scripts/python.mk @@ -31,36 +31,37 @@ pypi_test_upload_url := https://test.pypi.org/legacy/ # get the python package version # (evaluates to that string) -# parameter 1: name of the project (aka api, robot-server, etc) -# parameter 2: an extra version tag string -# parameter 3: override python_build_utils.py path (default: ../scripts/python_build_utils.py) +# parameter 1: name of the package (aka api, robot-server, etc) +# parameter 2: name of the project +# parameter 3: an extra version tag string +# parameter 4: override python_build_utils.py path (default: ../scripts/python_build_utils.py) define python_package_version -$(shell $(python) $(if $(3),$(3),../scripts/python_build_utils.py) $(1) normalize_version $(if $(2),-e $(2))) +$(shell $(python) $(if $(4),$(4),../scripts/python_build_utils.py) $(1) $(2) normalize_version $(if $(3),-e $(3))) endef # This is the poetry version of python_get_wheelname. Arguments are identical. define poetry_python_get_wheelname -$(2)-$(call python_package_version,$(1),$(3),$(4))-py3-none-any.whl +$(3)-$(call python_package_version,$(1),$(2),$(4),$(5))-py3-none-any.whl endef # get the name of the wheel that setup.py will build -# parameter 1: the name of the project (aka api, robot-server, etc) -# parameter 2: the name of the python package (aka opentrons, robot_server, etc) -# parameter 3: any extra version tags -# parameter 4: override python_build_utils.py path (default: ../scripts/python_build_utils.py) - +# parameter 1: the name of the package (aka api, robot-server, etc) +# parameter 2: the name of the project (aka robot-stack, ot3, etc) +# parameter 3: the name of the python package (aka opentrons, robot_server, etc) +# parameter 4: any extra version tags +# parameter 5: override python_build_utils.py path (default: ../scripts/python_build_utils.py) define python_get_wheelname -$(2)-$(call python_package_version,$(1),$(3),$(4))-py2.py3-none-any.whl +$(3)-$(call python_package_version,$(1),$(2),$(4),$(5))-py2.py3-none-any.whl endef # get the name of the sdist that setup.py will build # parameter 1: the name of the project (aka api, robot-server, etc) # parameter 2: the name of the python package (aka opentrons, robot_server, etc) -# parameter 3: any extra version tags -# parameter 4: override python_build_utils.py path (default: ../scripts/python_build_utils.py) - +# parameter 3: the name of the project (aka robot-stack, ot3, docs) +# parameter 4: any extra version tags +# parameter 5: override python_build_utils.py path (default: ../scripts/python_build_utils.py) define python_get_sdistname -$(2)-$(call python_package_version,$(1),$(3),$(4)).tar.gz +$(3)-$(call python_package_version,$(1),$(2),$(4),$(5)).tar.gz endef # upload a package to a repository diff --git a/scripts/python_build_utils.py b/scripts/python_build_utils.py index 3cc596a6be4..da491486f11 100644 --- a/scripts/python_build_utils.py +++ b/scripts/python_build_utils.py @@ -8,10 +8,12 @@ import json import os import subprocess +import sys from collections import namedtuple -PackageEntry = namedtuple("PackageEntry", ("pkg_json", "br_version_prefix")) +PackageEntry = namedtuple("PackageEntry", ("br_version_prefix")) +ProjectEntry = namedtuple("ProjectEntry", ("tag_prefix")) HERE = os.path.dirname(__file__) @@ -22,42 +24,31 @@ package_entries = { - 'api': PackageEntry( - os.path.join(HERE, '..', 'api', 'src', 'opentrons', 'package.json'), - 'opentrons_api'), - 'update-server': PackageEntry( - os.path.join(HERE, '..', 'update-server', 'otupdate', 'package.json'), - 'update_server'), - 'robot-server': PackageEntry( - os.path.join(HERE, '..', 'robot-server', 'robot_server', 'package.json'), - 'robot_server'), - 'shared-data': PackageEntry( - os.path.join(HERE, '..', 'shared-data', 'package.json'), - 'shared_data'), - 'notify-server': PackageEntry( - os.path.join(HERE, '..', 'notify-server', 'notify_server', 'package.json'), - 'notify_server'), - 'hardware': PackageEntry( - os.path.join(HERE, '..', 'hardware', 'opentrons_hardware', 'package.json'), - 'opentrons_hardware'), - 'usb-bridge': PackageEntry( - os.path.join(HERE, '..', 'usb-bridge', 'ot3usb', 'package.json'), - 'usb_bridge'), + 'api': PackageEntry('opentrons_api'), + 'update-server': PackageEntry('update_server'), + 'robot-server': PackageEntry('robot_server'), + 'shared-data': PackageEntry('shared_data'), + 'notify-server': PackageEntry('notify_server'), + 'hardware': PackageEntry('opentrons_hardware'), + 'usb-bridge': PackageEntry('usb_bridge'), +} + +project_entries = { + 'ot3': ProjectEntry('ot3@'), + 'robot-stack': ProjectEntry('v'), + 'docs': ProjectEntry('docs@'), } -def get_version(project, extra_tag=''): - pkg_json_path = package_entries[project].pkg_json - builtin_ver = json.load(open(pkg_json_path))['version'] +def get_version(package, project, extra_tag=''): + builtin_ver = _latest_version_for_project(project) if extra_tag: version = builtin_ver + '.dev{}'.format(extra_tag) else: version = builtin_ver return version - - -def normalize_version(project, extra_tag=''): +def normalize_version(package, project, extra_tag=''): # Pipenv requires setuptools >= 36.2.1. Since 36.2.1, setuptools changed # the way they vendor dependencies, like the packaging module that # provides the way to normalize version numbers for wheel file names. So @@ -68,9 +59,33 @@ def normalize_version(project, extra_tag=''): except ImportError: # old way from pkg_resources.extern import packaging - vers_obj = packaging.version.Version(get_version(project, extra_tag)) + vers_obj = packaging.version.Version(get_version(package, project, extra_tag)) return str(vers_obj) +def _latest_tag_for_prefix(prefix): + try: + tags_result = subprocess.check_output( + ['git', 'describe', '--tags', '--abbrev=0', '--match=' + prefix + '*'], + cwd=CWD) + except subprocess.CalledProcessError: + # This happens if a tag for the project didn't exist. This might be because + # this is a new project that hasn't been tagged yet; it also might be because + # this is being run outside the git repo, or possibly the repo was pulled + # without its tags. We'll print an error (to stderr, since this is called as + # a shell program by make and printing it to stdout would get captured). + sys.stderr.write( + 'Could not find tag matching {prefix} '.format(prefix=prefix) + + '- build before release or no tags. Using 0.0.0-dev\n') + tags_result = prefix.encode() + b'0.0.0-dev' + tags_matching = tags_result.strip().split(b'\n') + + + return tags_matching[-1].decode() + +def _latest_version_for_project(project): + prefix = project_entries[project].tag_prefix + tag = _latest_tag_for_prefix(prefix) + return tag.split(prefix)[1] def _ref_from_sha(sha): # codebuild leaves us in detached HEAD, so we need to pull some @@ -104,31 +119,32 @@ def _ref_from_sha(sha): return sha[:12] -def dump_br_version(project, extra_tag=''): +def dump_br_version(package, project, extra_tag=''): """ Dump an enhanced version json including - - The version from package.json + - The version from the latest git tag - The current branch (if it can be found) - The current sha """ - normalized = get_version(project, extra_tag) + normalized = get_version(package, project, extra_tag) sha = subprocess.check_output( ['git', 'rev-parse', 'HEAD'], cwd=CWD).strip().decode() branch = _ref_from_sha(sha) - pref = package_entries[project].br_version_prefix + pref = package_entries[package].br_version_prefix return json.dumps({pref+'_version': normalized, pref+'_sha': sha, pref+'_branch': branch}) - if __name__ == '__main__': parser = argparse.ArgumentParser( description='Perform one of several build-time tasks') - parser.add_argument(dest='project', metavar='SUBPROJECT', type=str, + parser.add_argument(dest='package', metavar='PACKAGE', type=str, choices=package_entries.keys()) + parser.add_argument(dest='project', metavar='PROJECT', type=str, + choices=project_entries.keys()) parser.add_argument(dest='task', metavar='TASK', type=str, - choices=['normalize_version', 'dump_br_version']) + choices=['normalize_version', 'dump_br_version', 'get_version']) parser.add_argument('-e', '--extra-tag', type=str, default='', help='Extra version tag like a build number', dest='extra_tag') args = parser.parse_args() - print(locals()[args.task](args.project, args.extra_tag)) + print(locals()[args.task](args.package, args.project, args.extra_tag)) diff --git a/shared-data/package.json b/shared-data/package.json index 2da46a56e92..85605a4e3d8 100755 --- a/shared-data/package.json +++ b/shared-data/package.json @@ -1,6 +1,6 @@ { "name": "@opentrons/shared-data", - "version": "6.2.0", + "version": "0.0.0-dev", "description": "Default labware definitions for Opentrons robots", "repository": { "type": "git", diff --git a/shared-data/python/Makefile b/shared-data/python/Makefile index 734564400c6..3087b0b765a 100644 --- a/shared-data/python/Makefile +++ b/shared-data/python/Makefile @@ -20,6 +20,10 @@ PATH := $(shell cd ../.. && yarn bin):$(PATH) # documentation BUILD_NUMBER ?= +ot_project := $(OPENTRONS_PROJECT) +project_rs_default = $(if $(ot_project),$(ot_project),robot-stack) +project_ot3_default = $(if $(ot_project),$(ot_project),ot3) + # this may be set as an environment variable to select the version of # python to run if pyenv is not available. it should always be set to # point to a python3.6. @@ -28,8 +32,8 @@ OT_PYTHON ?= python BUILD_DIR := dist -wheel_file = $(BUILD_DIR)/$(call python_get_wheelname,shared-data,opentrons_shared_data,$(BUILD_NUMBER),../../scripts/python_build_utils.py) -sdist_file = $(BUILD_DIR)/$(call python_get_sdistname,shared-data,opentrons_shared_data,,../../scripts/python_build_utils.py) +wheel_file = $(BUILD_DIR)/$(call python_get_wheelname,shared-data,$(project_rs_default),opentrons_shared_data,$(BUILD_NUMBER),../../scripts/python_build_utils.py) +sdist_file = $(BUILD_DIR)/$(call python_get_sdistname,shared-data,$(project_ot3_default),opentrons_shared_data,,../../scripts/python_build_utils.py) py_sources = $(filter %.py,$(shell $(SHX) find opentrons_shared_data)) opentrons_shared_data/py.typed deck_sources = $(wildcard ../deck/definitions/*/*.json) $(wildcard ../deck/schemas/*.json) @@ -80,12 +84,14 @@ wheel: $(wheel_file) sdist: $(sdist_file) +$(wheel_file): export OPENTRONS_PROJECT=$(project_rs_default) $(wheel_file): setup.py $(py_sources) $(json_sources) $(SHX) mkdir -p build $(python) setup.py $(wheel_opts) bdist_wheel $(SHX) ls $(BUILD_DIR) +$(sdist_file): export OPENTRONS_PROJECT=$(project_ot3_default) $(sdist_file): setup.py $(py_sources) $(json_sources) $(SHX) mkdir -p build $(python) setup.py sdist diff --git a/shared-data/python/buildroot.mk b/shared-data/python/buildroot.mk index edd1e15a7f5..cbf35dc8393 100644 --- a/shared-data/python/buildroot.mk +++ b/shared-data/python/buildroot.mk @@ -4,25 +4,20 @@ # ################################################################################ -# Get a key from package.json (like version) -define get_pkg_json_key - $(shell python -c "import json; print(json.load(open('$(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/shared-data/package.json'))[\"$(1)\"])") +define OTSHAREDDATA_CALL_PBU + $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py shared-data robot-stack $(1)) endef -PYTHON_OPENTRONS_SHARED_DATA_VERSION = $(call get_pkg_json_key,version) -PYTHON_OPENTRONS_SHARED_DATA_LICENSE = $(call get_pkg_json_key,license) +PYTHON_OPENTRONS_SHARED_DATA_VERSION = $(call OTSHAREDDATA_CALL_PBU,get_version) +PYTHON_OPENTRONS_SHARED_DATA_LICENSE = Apache-2 PYTHON_OPENTRONS_SHARED_DATA_LICENSE_FILES = $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/LICENSE PYTHON_OPENTRONS_SHARED_DATA_SETUP_TYPE = setuptools PYTHON_OPENTRONS_SHARED_DATA_SITE_METHOD = local PYTHON_OPENTRONS_SHARED_DATA_SITE = $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH) PYTHON_OPENTRONS_SHARED_DATA_SUBDIR = shared-data/python -define OTSD_DUMP_BR_VERSION - $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py shared-data dump_br_version) -endef - define PYTHON_OPENTRONS_SHARED_DATA_INSTALL_VERSION - echo '$(call OTSD_DUMP_BR_VERSION)' > $(BINARIES_DIR)/opentrons-shared-data-version.json + echo '$(call OTSHAREDDATA_CALL_PBU,dump_br_version)' > $(BINARIES_DIR)/opentrons-shared-data-version.json endef ot_sd_name := python-opentrons-shared-data diff --git a/shared-data/python/opentrons_shared_data/__init__.py b/shared-data/python/opentrons_shared_data/__init__.py index 1428efb1353..dfe23b0a638 100644 --- a/shared-data/python/opentrons_shared_data/__init__.py +++ b/shared-data/python/opentrons_shared_data/__init__.py @@ -5,18 +5,13 @@ """ import os -import json from .load import get_shared_data_root, load_shared_data HERE = os.path.abspath(os.path.dirname(__file__)) -try: - with open(os.path.join(HERE, "package.json")) as pkg: - package_json = json.load(pkg) - __version__ = package_json.get("version") -except (FileNotFoundError, OSError): - __version__ = "unknown" +from ._version import version # noqa: E402 +__version__ = version __all__ = ["__version__", "get_shared_data_root", "load_shared_data"] diff --git a/shared-data/python/opentrons_shared_data/_version.py b/shared-data/python/opentrons_shared_data/_version.py new file mode 100644 index 00000000000..591ff81b627 --- /dev/null +++ b/shared-data/python/opentrons_shared_data/_version.py @@ -0,0 +1,27 @@ +import sys +import logging + +_pyversion = sys.version_info[0:2] +if _pyversion < (3, 7): + raise RuntimeError( + "opentrons requires Python 3.7 or above, this is {0}.{1}".format( + _pyversion[0], _pyversion[1] + ) + ) + + +if _pyversion >= (3, 8): + from importlib import metadata +else: + import importlib_metadata as metadata # type: ignore[no-redef] + +try: + version: str = metadata.version("opentrons_shared_data") # type: ignore[attr-defined] +except Exception as e: + logging.warning( + "Could not determine version for opentrons, may be dev install, using 0.0.0-dev" + ) + logging.debug(e) + version = "0.0.0-dev" + +__all__ = ["version"] diff --git a/shared-data/python/setup.py b/shared-data/python/setup.py index 8cc3912b02d..0c3924d45f1 100644 --- a/shared-data/python/setup.py +++ b/shared-data/python/setup.py @@ -63,11 +63,6 @@ def make_release_tree(self, base_dir, files) -> None: msg=f"copying and minimizing {data_file} -> {target_file}", ) - # also grab our package.json - self.copy_file( - os.path.join(HERE, "..", "package.json"), - os.path.join(base_dir, "package.json"), - ) super().make_release_tree(base_dir, files) @@ -88,11 +83,10 @@ def _get_data_files(self): # We want a list of paths to only files relative to ../shared-data to_include = [str(f.relative_to(DATA_ROOT)) for f in get_shared_data_files()] destination = os.path.join(build_base, "opentrons_shared_data", DEST_BASE_PATH) - # And finally, tell the system about our files, including package.json + # And finally, tell the system about our files files.extend( [ ("opentrons_shared_data", DATA_ROOT, destination, to_include), - ("opentrons_shared_data", "..", build_base, ["package.json"]), ] ) return files @@ -100,11 +94,12 @@ def _get_data_files(self): def get_version(): buildno = os.getenv("BUILD_NUMBER") + project = os.getenv("OPENTRONS_PROJECT", "robot-stack") if buildno: normalize_opts = {"extra_tag": buildno} else: normalize_opts = {} - return normalize_version("shared-data", **normalize_opts) + return normalize_version("shared-data", project, **normalize_opts) VERSION = get_version() diff --git a/update-server/MANIFEST.in b/update-server/MANIFEST.in deleted file mode 100644 index 756f66b8e53..00000000000 --- a/update-server/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include otupdate/package.json diff --git a/update-server/Makefile b/update-server/Makefile index 4c124a967ef..150dc8f9d13 100644 --- a/update-server/Makefile +++ b/update-server/Makefile @@ -10,12 +10,15 @@ SHX := npx shx # add yarn CLI dev deps to PATH (for cross platform POSIX commands via shx) PATH := $(shell cd .. && yarn bin):$(PATH) +ot_project := $(OPENTRONS_PROJECT) +project_rs_default = $(if $(ot_project),$(ot_project),robot-stack) +project_ot3_default = $(if $(ot_project),$(ot_project),ot3) port ?= 34000 tests ?= tests test_opts ?= -wheel_file = $(call python_get_wheelname,update-server,otupdate) -sdist_file = $(call python_get_sdistname,update-server,otupdate) +wheel_file = $(call python_get_wheelname,update-server,$(project_rs_default),otupdate) +sdist_file = $(call python_get_sdistname,update-server,$(project_ot3_default),otupdate) # Host key location for buildroot robot br_ssh_key ?= $(default_ssh_key) # Other SSH args for buildroot robots @@ -62,12 +65,14 @@ format: $(python) -m black otupdate tests .PHONY: wheel +wheel: export OPENTRONS_PROJECT=$(project_rs_default) wheel: clean $(python) setup.py bdist_wheel $(SHX) rm -rf build $(SHX) ls dist .PHONY: sdist +sdist: export OPENTRONS_PROJECT=$(project_ot3_default) sdist: clean $(python) setup.py sdist $(SHX) rm -rf build diff --git a/update-server/buildroot.mk b/update-server/buildroot.mk index 53650b9daaf..f9a7224253b 100644 --- a/update-server/buildroot.mk +++ b/update-server/buildroot.mk @@ -3,13 +3,13 @@ # python-opentrons-update-server # ################################################################################ -# Get a key from package.json (like version) -define get_pkg_json_key - $(shell python -c "import json; print(json.load(open('$(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/update-server/otupdate/package.json'))[\"$(1)\"])") + +define OTUPDATESERVER_CALL_PBU + $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py update-server robot-stack $(1)) endef -PYTHON_OPENTRONS_UPDATE_SERVER_VERSION = $(call get_pkg_json_key,version) -PYTHON_OPENTRONS_UPDATE_SERVER_LICENSE = $(call get_pkg_json_key,license) +PYTHON_OPENTRONS_UPDATE_SERVER_VERSION = $(call OTUPDATESERVER_CALL_PBU,get_version) +PYTHON_OPENTRONS_UPDATE_SERVER_LICENSE = Apache-2 PYTHON_OPENTRONS_UPDATE_SERVER_LICENSE_FILES = $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/LICENSE PYTHON_OPENTRONS_UPDATE_SERVER_SETUP_TYPE = setuptools PYTHON_OPENTRONS_UPDATE_SERVER_SITE_METHOD = local @@ -18,11 +18,11 @@ PYTHON_OPENTRONS_UPDATE_SERVER_SUBDIR = update-server PYTHON_OPENTRONS_UPDATE_SERVER_POST_INSTALL_TARGET_HOOKS = PYTHON_OPENTRONS_UPDATE_SERVER_INSTALL_VERSION define OTUS_DUMP_BR_VERSION - $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py update-server dump_br_version) + $(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py update-server robot-stack dump_br_version) endef define PYTHON_OPENTRONS_UPDATE_SERVER_INSTALL_VERSION - echo '$(call OTUS_DUMP_BR_VERSION)' > $(BINARIES_DIR)/opentrons-update-server-version.json + echo '$(call OTUPDATESERVER_CALL_PBU,dump_br_version)' > $(BINARIES_DIR)/opentrons-update-server-version.json endef otupdate_name := python-opentrons-update-server diff --git a/update-server/otupdate/package.json b/update-server/otupdate/package.json deleted file mode 100644 index f87cfc64282..00000000000 --- a/update-server/otupdate/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@opentrons/update-server", - "version": "6.2.0", - "description": "Update server for Opentrons robots", - "repository": { - "type": "git", - "url": "https://github.com/Opentrons/opentrons.git" - }, - "author": { - "name": "Opentrons Labworks", - "email": "engineering@opentrons.com" - }, - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/Opentrons/opentrons/issues" - }, - "homepage": "https://github.com/Opentrons/opentrons" -} diff --git a/update-server/readme-updateserver-for-OE b/update-server/readme-updateserver-for-OE deleted file mode 100644 index 189ff4326e4..00000000000 --- a/update-server/readme-updateserver-for-OE +++ /dev/null @@ -1,2 +0,0 @@ -Refactoring and adding on top of current update-server, making it usable across OE and buildroot - diff --git a/update-server/setup.py b/update-server/setup.py index 4454e9a7639..b91bd4358a6 100644 --- a/update-server/setup.py +++ b/update-server/setup.py @@ -4,14 +4,22 @@ import os from setuptools import setup, find_packages import json +import sys HERE = os.path.abspath(os.path.dirname(__file__)) +sys.path.append(os.path.join(HERE, "..", "scripts")) + +from python_build_utils import normalize_version # noqa: E402 def get_version(): - with open(os.path.join(HERE, 'otupdate', 'package.json')) as pkg: - package_json = json.load(pkg) - return package_json.get('version') + buildno = os.getenv("BUILD_NUMBER") + project = os.getenv('OPENTRONS_PROJECT', 'robot-stack') + if buildno: + normalize_opts = {"extra_tag": buildno} + else: + normalize_opts = {} + return normalize_version("update-server", project, **normalize_opts) VERSION = get_version() diff --git a/update-server/updateServerDesignNotes.md b/update-server/updateServerDesignNotes.md deleted file mode 100644 index 531a8da2068..00000000000 --- a/update-server/updateServerDesignNotes.md +++ /dev/null @@ -1,52 +0,0 @@ -# Update server design review - -## Problem - -Currently, update endpoint is in API server, and it does not do any safety -checking to ensure that the newly installed version will be able to come up -and accept successive updates. In this scenario, the API can go down -unrecoverably and require difficult intervention from engineering to get -the robot back online. - - -## Proposed solution - -Create a separate server that will handle updates of SW, FW, and configs. - -This server needs to: -- report current versions of all relevant software installed on the robot - - minimally: self, API server, Smoothie FW - - optionally: module FW, Python, ResinOS, Docker container (as available) -- accept updates to itself and install them safely. When it gets an update: - - Create a virtual environment - - Install the update into the virtual environment - - Start the server on another port - - Test the `health` and self-update endpoints in the new server (be - careful here about recursion) - - Check test status. If success, install over top of self and restart. - If failure, report status and delete virtual environment -- accept updates to the API server and install them (safety checks optional - for now) -- accept updates to Smoothie and module FW and install them (safety checks - optional for now) -- provide other endpoints for modifying configurations such as feature-flags - and robot configs - -For backward compatibility, core update endpoints must also be supported in -the API server, in case the update server is not present. Shared -implementations should live in "api-server-lib". Imports of `opentrons` -should be limited, and wrapped in a `try...except` block with graceful -failure (in general, all care should be taken to ensure that this server -*can not* fail in its core task: bootstrapping itself from any prior -version). - -Steps for manually testing (warning: this can modify your system install of `otupdate`): -- run `make dev` with the "version" in package.json unchanged (this starts the update server on http://127.0.0.1:34000) -- change "version" to a different value and `make wheel` in another terminal -- use Postman or curl to check the health endpoint - - with curl: `curl http://127.0.0.1:34000/server/update/health` - - result should be a json packet including the server name and versions for update server, API server, and Smoothie -- use Postman or curl to send the new wheel to the bootstrap endpoint - - with curl: `curl -X POST -H "Content-Type: multipart/form-data" -F "whl=@/path/to/otupdate-.whl" http://127.0.0.1:34000/server/update/bootstrap` - - this will take a while, but it should eventually return a success message and then the installed version of update server on your system should be the new version string - diff --git a/usb-bridge/Makefile b/usb-bridge/Makefile index cd162f0b323..dd24f5456bc 100644 --- a/usb-bridge/Makefile +++ b/usb-bridge/Makefile @@ -12,13 +12,16 @@ SHX := npx shx # add yarn CLI dev deps to PATH (for cross platform POSIX commands via shx) PATH := $(shell cd .. && yarn bin):$(PATH) +ot_project := $(OPENTRONS_PROJECT) +project_ot3_default = $(if $(ot_project),$(ot_project),ot3) + # These variables can be overriden when make is invoked to customize the # behavior of pytest. For instance, # make test tests=tests/ot3usb/abcd.py would run only the # specified test tests ?= tests test_opts ?= -sdist_file = $(call python_get_sdistname,usb-bridge,ot3usb) +sdist_file = $(call python_get_sdistname,usb-bridge,$(project_ot3_default),ot3usb) # Host key location for buildroot robot br_ssh_key ?= $(default_ssh_key) # Other SSH args for buildroot robots @@ -59,6 +62,7 @@ format: $(python) -m black ot3usb tests .PHONY: sdist +sdist: export OPENTRONS_PROJECT=$(project_ot3_default) sdist: clean $(python) setup.py sdist $(SHX) rm -rf build diff --git a/usb-bridge/ot3usb/package.json b/usb-bridge/ot3usb/package.json deleted file mode 100644 index 75f7907cd68..00000000000 --- a/usb-bridge/ot3usb/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@opentrons/update-server", - "version": "0.0.1", - "description": "USB-TCP bridge for Opentrons OT-3 robots", - "repository": { - "type": "git", - "url": "https://github.com/Opentrons/opentrons.git" - }, - "author": { - "name": "Opentrons Labworks", - "email": "engineering@opentrons.com" - }, - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/Opentrons/opentrons/issues" - }, - "homepage": "https://github.com/Opentrons/opentrons" -} diff --git a/usb-bridge/setup.py b/usb-bridge/setup.py index ae10afc5c0d..97c0f505951 100644 --- a/usb-bridge/setup.py +++ b/usb-bridge/setup.py @@ -4,14 +4,22 @@ import os from setuptools import setup, find_packages import json +import sys HERE = os.path.abspath(os.path.dirname(__file__)) +sys.path.append(os.path.join(HERE, "..", "scripts")) + +from python_build_utils import normalize_version # noqa: E402 def get_version(): - with open(os.path.join(HERE, 'ot3usb', 'package.json')) as pkg: - package_json = json.load(pkg) - return package_json.get('version') + buildno = os.getenv("BUILD_NUMBER") + project = os.getenv('OPENTRONS_PROJECT', 'robot-stack') + if buildno: + normalize_opts = {"extra_tag": buildno} + else: + normalize_opts = {} + return normalize_version('usb-bridge', project, **normalize_opts) VERSION = get_version()