refactor(build): find py versions from git

The versions for python packages are now injected from the latest git
tag detected in the repo history, rather than from an otherwise-useless
package.json. This is dependent on the project - for packages used in
multiple projects, you have to decide what project
you're building for. That's done right now through an environment
variable. Happily enough, there's only two major projects that build
python packages and they use different distribution formats and thus
different makefile recipes, and thus can have different defaults.

This also requires modifying a lot of the internals of how we do
versions and while I was at it I went back to using phony targets in
makefiles for wheels and sdists because yeah it means you have to
rebuild them but it also makes the makefile way way way faster to parse.

The now-unused package.jsons are removed, and since yarn now doesn't
need to know about the python projects and they don't have package.json
anymore they're pulled out of the workspace list.

Sadly enough, this change means that any workflow that needs to produce
versioned artifacts is going to need git history.

.github/workflows/api-test-lint-deploy.yaml

@@ -129,6 +129,8 @@ jobs:
     if: github.event_name == 'push'
     steps:
       - uses: 'actions/checkout@v3'
+        with:
+          fetch-depth: 0
       - uses: 'actions/setup-node@v3'
         with:
           node-version: '16'

.github/workflows/docs-build.yaml

@@ -41,6 +41,8 @@ jobs:
     runs-on: 'ubuntu-22.04'
     steps:
       - uses: 'actions/checkout@v3'
+        with:
+          fetch-depth: 0
       - uses: 'actions/setup-node@v3'
         with:
           node-version: '16'

.github/workflows/hardware-lint-test.yaml

@@ -45,7 +45,7 @@ jobs:
     steps:
       - name: Checkout opentrons repo
         uses: 'actions/checkout@v3'
-
+        fetch-depth: 0
       - name: Setup Node
         uses: 'actions/setup-node@v3'
         with:

.github/workflows/hardware-testing-protocols.yaml

@@ -39,6 +39,7 @@ jobs:
     steps:
       - name: Checkout opentrons repo
         uses: 'actions/checkout@v3'
+        fetch-depth: 0
 
       - name: Setup Node
         uses: 'actions/setup-node@v3'

.github/workflows/shared-data-test-lint-deploy.yaml

@@ -144,6 +144,8 @@ jobs:
     if: github.event_name == 'push'
     steps:
       - uses: 'actions/checkout@v3'
+        with:
+          fetch-depth: 0
       - uses: 'actions/setup-node@v3'
         with:
           node-version: '16'

api/MANIFEST.in

@@ -1,4 +1,3 @@
-include src/opentrons/package.json
 graft src/opentrons/config
 graft src/opentrons/resources
 exclude pyproject.toml

api/Makefile

@@ -17,14 +17,17 @@ sphinx_build := $(pipenv) run sphinx-build -W --keep-going
 # github.com/Opentrons/opentrons/issues/6135
 sphinx_build_allow_warnings := $(pipenv) run sphinx-build
 
+ot_project := $(OPENTRONS_PROJECT)
+project_rs_default = $(if $(ot_project),$(ot_project),robot-stack)
+project_ot3_default = $(if $(ot_project),$(ot_project),ot3)
 
-# Find the version of the wheel from package.json using a helper script. We
+# Find the version of the wheel from git using a helper script. We
 # use python here so we can use the same version normalization that will be
 # used to create the wheel.
-wheel_file = dist/$(call python_get_wheelname,api,opentrons,$(BUILD_NUMBER))
+wheel_file = dist/$(call python_get_wheelname,api,$(project_rs_default),opentrons,$(BUILD_NUMBER))
 
-# Find the version of the sdist file from package.json using a helper script.
-sdist_file = dist/$(call python_get_sdistname,api,opentrons)
+# Find the version of the sdist file from git using a helper script.
+sdist_file = dist/$(call python_get_sdistname,api,$(project_ot3_default),opentrons)
 
 # These variables are for simulating python protocols
 sim_log_level ?= info
@@ -51,18 +54,12 @@ ssh_opts ?= $(default_ssh_opts)
 twine_auth_args := --username $(pypi_username) --password $(pypi_password)
 twine_repository_url ?= $(pypi_test_upload_url)
 
-# Source discovery
-# For the python sources
-ot_py_sources := $(filter %.py,$(shell $(SHX) find src/opentrons/))
-# And the out of tree shared data
-ot_shared_data_sources := $(filter %.json,$(shell $(SHX) find ../shared-data/))
-# And the arbitrary stuff in resources
-ot_resources := $(filter %,$(shell $(SHX) find src/opentrons/resources))
-ot_sources := $(ot_py_sources) $(ot_shared_data_sources) $(ot_resources)
-
 # Defined separately than the clean target so the wheel file doesn’t have to
 # depend on a PHONY target
-clean_cmd = $(SHX) rm -rf build dist .coverage coverage.xml '*.egg-info' '**/__pycache__' '**/*.pyc' 'src/**/.mypy_cache'
+clean_cmd = $(SHX) rm -rf build .coverage coverage.xml '*.egg-info' '**/__pycache__' '**/*.pyc' 'src/**/.mypy_cache'
+clean_wheel_cmd = $(clean_cmd) dist/*.whl
+clean_sdist_cmd = $(clean_cmd) dist/*.tar.gz
+clean_all_cmd = $(clean_cmd) dist
 
 plot_type ?=
 
@@ -82,28 +79,29 @@ setup-ot2:
 
 .PHONY: clean
 clean: docs-clean
-	$(clean_cmd)
+	$(clean_all_cmd)
 
 .PHONY: teardown
 teardown:
 	$(pipenv) --rm
 
-$(wheel_file): setup.py $(ot_sources)
-	$(clean_cmd)
+.PHONY: wheel
+wheel: export OPENTRONS_PROJECT=$(project_rs_default)
+wheel:
+	$(clean_wheel_cmd)
 	$(python) setup.py $(wheel_opts) bdist_wheel
 	$(SHX) rm -rf build
 	$(SHX) ls dist
 
-wheel: $(wheel_file)
 
-$(sdist_file): setup.py $(ot_sources)
-	$(clean_cmd)
+.PHONY: sdist
+sdist: export OPENTRONS_PROJECT=$(project_ot3_default)
+sdist:
+	$(clean_sdist_cmd)
 	$(python) setup.py sdist
 	$(SHX) rm -rf build
 	$(SHX) ls dist
 
-sdist: $(sdist_file)
-
 .PHONY: test
 test:
 	$(pytest) $(tests) $(test_opts)

api/buildroot.mk

@@ -4,26 +4,21 @@
 #
 ################################################################################
 
-# Get a key from package.json (like version)
-define get_pkg_json_key
-	$(shell python -c "import json; print(json.load(open('$(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/api/src/opentrons/package.json'))[\"$(1)\"])")
+define OTAPI_CALL_PBU
+	$(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py api robot-stack $(1))
 endef
 
-PYTHON_OPENTRONS_API_VERSION = $(call get_pkg_json_key,version)
-PYTHON_OPENTRONS_API_LICENSE = $(call get_pkg_json_key,license)
+PYTHON_OPENTRONS_API_VERSION = $(call OTAPI_CALL_PBU,get_version)
+PYTHON_OPENTRONS_API_LICENSE = Apache-2
 PYTHON_OPENTRONS_API_LICENSE_FILES = $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/LICENSE
 PYTHON_OPENTRONS_API_SETUP_TYPE = setuptools
 PYTHON_OPENTRONS_API_SITE_METHOD = local
 PYTHON_OPENTRONS_API_SITE = $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)
 PYTHON_OPENTRONS_API_SUBDIR = api
 PYTHON_OPENTRONS_API_POST_INSTALL_TARGET_HOOKS = PYTHON_OPENTRONS_API_INSTALL_VERSION PYTHON_OPENTRONS_API_INSTALL_RELEASE_NOTES
 
-define OTAPI_DUMP_BR_VERSION
-	$(shell python $(BR2_EXTERNAL_OPENTRONS_MONOREPO_PATH)/scripts/python_build_utils.py api dump_br_version)
-endef
-
 define PYTHON_OPENTRONS_API_INSTALL_VERSION
-	echo '$(call OTAPI_DUMP_BR_VERSION)' > $(BINARIES_DIR)/opentrons-api-version.json
+	echo '$(call OTAPI_CALL_PBU,dump_br_version)' > $(BINARIES_DIR)/opentrons-api-version.json
 endef
 
 ot_api_name := python-opentrons-api

api/docs/hardware/conf.py

@@ -21,9 +21,6 @@
 import sys
 import json
 import pkgutil
-# This load means sphinx docs builds must be run in a virtualenv with the
-# newly-built module
-_package_json = json.loads(pkgutil.get_data('opentrons', 'package.json'))
 
 sys.path.insert(0, os.path.abspath('../..'))
 sys.path.insert(0, os.path.abspath('../sphinxext'))
@@ -66,16 +63,20 @@
 # General information about the project.
 project = 'OT-2 Hardware Control API'
 copyright = '2020, Opentrons'
-author = _package_json['author']['name']
+author = 'Opentrons Labworks'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
-#
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts'))
+import python_build_utils
+sys.path = sys.path[:-1]
+_vers = python_build_utils.get_version('api', 'robot-stack')
+
 # The short X.Y version.
-version = '.'.join(_package_json['version'].split('.')[:2])
+version = '.'.join(_vers.split('.')[:2])
 # The full version, including alpha/beta/rc tags.
-release = _package_json['version']
+release = _vers
 
 # setup the code block substitution extension to auto-update apiLevel
 extensions += ['sphinx-prompt', 'sphinx_substitution_extensions']
@@ -320,7 +321,7 @@
 latex_documents = [
     (master_doc, 'OT2HardwareControl.tex',
      'OT-2 Hardware Control API Documentation',
-     _package_json['author']['name'], 'howto'),
+     'Opentrons Labworks', 'howto'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of

api/docs/v1/conf.py

@@ -21,9 +21,6 @@
 import sys
 import json
 import pkgutil
-# This load means sphinx docs builds must be run in a virtualenv with the
-# newly-built module
-_package_json = json.loads(pkgutil.get_data('opentrons', 'package.json'))
 
 sys.path.insert(0, os.path.abspath('./api_cache'))
 sys.path.insert(0, os.path.abspath('../..'))
@@ -67,16 +64,21 @@
 # General information about the project.
 project = 'OT-2 API V1'
 copyright = '2019, Opentrons'
-author = _package_json['author']['name']
+author = "Opentrons Labworks"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
-#
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts'))
+import python_build_utils
+sys.path = sys.path[:-1]
+_vers = python_build_utils.get_version('api', 'robot-stack')
+
 # The short X.Y version.
-version = '.'.join(_package_json['version'].split('.')[:2])
+version = '.'.join(_vers.split('.')[:2])
 # The full version, including alpha/beta/rc tags.
-release = _package_json['version']
+release = _vers
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -311,7 +313,7 @@
 latex_documents = [
     (master_doc,
      'OpentronsPythonAPIV1.tex', 'OT-2 Python API V1 Documentation',
-     _package_json['author']['name'], 'howto'),
+     'Opentrons Labworks', 'howto'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of

api/docs/v2/conf.py

@@ -21,9 +21,6 @@
 import sys
 import json
 import pkgutil
-# This load means sphinx docs builds must be run in a virtualenv with the
-# newly-built module
-_package_json = json.loads(pkgutil.get_data('opentrons', 'package.json'))
 
 sys.path.insert(0, os.path.abspath('../..'))
 sys.path.insert(0, os.path.abspath('../sphinxext'))
@@ -74,19 +71,25 @@
 # General information about the project.
 project = 'OT-2 API V2'
 copyright = '2010, Opentrons'
-author = _package_json['author']['name']
+author = 'Opentrons Labworks'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # todo(mm, 2021-09-30): Depending on where these show up, would it be more correct
 # to use the latest-supported *apiLevel* instead of the *Python package version*?
-#
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts'))
+import python_build_utils
+sys.path = sys.path[:-1]
+_vers = python_build_utils.get_version('api', 'robot-stack')
+
+
 # The short X.Y version.
-version = '.'.join(_package_json['version'].split('.')[:2])
+version = '.'.join(_vers.split('.')[:2])
 # The full version, including alpha/beta/rc tags.
-release = _package_json['version']
+release = _vers
 
 # setup the code block substitution extension to auto-update apiLevel
 extensions += ['sphinx-prompt', 'sphinx_substitution_extensions']
@@ -333,7 +336,7 @@
 latex_documents = [
     (master_doc, 'OpentronsPythonAPIV2.tex',
      'Opentrons Python API V2 Documentation',
-     _package_json['author']['name'], 'howto'),
+     'Opentrons Labworks', 'howto'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of

api/setup.py

@@ -22,11 +22,12 @@
 
 def get_version():
     buildno = os.getenv("BUILD_NUMBER")
+    project = os.getenv("OPENTRONS_PROJECT", "robot-stack")
     if buildno:
         normalize_opts = {"extra_tag": buildno}
     else:
         normalize_opts = {}
-    return normalize_version("api", **normalize_opts)
+    return normalize_version("api", project, **normalize_opts)
 
 
 VERSION = get_version()
@@ -66,6 +67,7 @@ def get_version():
     "pyserial==3.5",
     "typing-extensions>=4.0.0,<5",
     "click>=8.0.0,<9",
+    'importlib-metadata >= 1.0 ; python_version < "3.8"',
 ]
 
 
@@ -97,7 +99,7 @@ def read(*parts):
         install_requires=INSTALL_REQUIRES,
         include_package_data=True,
         package_dir={"": "src"},
-        package_data={"opentrons": ["py.typed", "package.json"]},
+        package_data={"opentrons": ["py.typed"]},
         entry_points={
             "console_scripts": [
                 "opentrons_simulate = opentrons.simulate:main",

api/src/opentrons/__init__.py

@@ -1,6 +1,5 @@
-import json
 import os
-import sys
+
 from pathlib import Path
 import logging
 import asyncio
@@ -25,28 +24,17 @@
 from opentrons.protocols.types import ApiDeprecationError
 from opentrons.protocols.api_support.types import APIVersion
 
-version = sys.version_info[0:2]
-if version < (3, 7):
-    raise RuntimeError(
-        "opentrons requires Python 3.7 or above, this is {0}.{1}".format(
-            version[0], version[1]
-        )
-    )
-
 HERE = os.path.abspath(os.path.dirname(__file__))
+from ._version import version  # noqa: E402
 
-try:
-    with open(os.path.join(HERE, "package.json")) as pkg:
-        package_json = json.load(pkg)
-        __version__ = package_json.get("version")
-except (FileNotFoundError, OSError):
-    __version__ = "unknown"
+__version__ = version
 
 from opentrons import config  # noqa: E402
 
 LEGACY_MODULES = ["robot", "reset", "instruments", "containers", "labware", "modules"]
 
-__all__ = ["version", "HERE", "config"]
+
+__all__ = ["version", "__version__", "HERE", "config"]
 
 
 def __getattr__(attrname: str) -> None:
@@ -155,7 +143,7 @@ async def initialize() -> ThreadManagedHardware:
     robot_conf = robot_configs.load()
     logging_config.log_init(robot_conf.log_level)
 
-    log.info(f"API server version: {__version__}")
+    log.info(f"API server version: {version}")
     log.info(f"Robot Name: {name()}")
 
     hardware = await _create_thread_manager()