Skip to content
This repository has been archived by the owner on Sep 13, 2023. It is now read-only.
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: iterative/mlem
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 60824ef5b9a20862e44e42ad0e338707050ef033
Choose a base ref
..
head repository: iterative/mlem
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 3d51c66f7f74ce48923be814d74d9720edd9b2f4
Choose a head ref
Showing with 6,826 additions and 2,052 deletions.
  1. +5 −1 .github/workflows/check-test-release.yml
  2. +4 −3 .pylintrc
  3. +0 −2 mlem/api/__init__.py
  4. +42 −81 mlem/api/commands.py
  5. +39 −5 mlem/api/utils.py
  6. +1 −2 mlem/cli/__init__.py
  7. +150 −75 mlem/cli/apply.py
  8. +70 −27 mlem/cli/build.py
  9. +8 −11 mlem/cli/checkenv.py
  10. +2 −14 mlem/cli/clone.py
  11. +13 −15 mlem/cli/config.py
  12. +189 −33 mlem/cli/declare.py
  13. +140 −62 mlem/cli/deployment.py
  14. +5 −3 mlem/cli/dev.py
  15. +1 −18 mlem/cli/import_object.py
  16. +3 −77 mlem/cli/info.py
  17. +8 −9 mlem/cli/init.py
  18. +7 −12 mlem/cli/link.py
  19. +301 −228 mlem/cli/main.py
  20. +64 −18 mlem/cli/serve.py
  21. +83 −63 mlem/cli/types.py
  22. +630 −0 mlem/cli/utils.py
  23. +20 −14 mlem/config.py
  24. +4 −1 mlem/constants.py
  25. +30 −5 mlem/contrib/bitbucketfs.py
  26. +7 −0 mlem/contrib/callable.py
  27. +10 −1 mlem/contrib/catboost.py
  28. +3 −1 mlem/contrib/docker/__init__.py
  29. +165 −133 mlem/contrib/docker/base.py
  30. +28 −20 mlem/contrib/docker/context.py
  31. +1 −0 mlem/contrib/docker/copy.j2
  32. +3 −6 mlem/contrib/docker/dockerfile.j2
  33. +4 −0 mlem/contrib/docker/install_req.j2
  34. +13 −4 mlem/contrib/dvc.py
  35. +9 −0 mlem/contrib/fastapi.py
  36. +11 −10 mlem/contrib/github.py
  37. +13 −6 mlem/contrib/gitlabfs.py
  38. +5 −0 mlem/contrib/heroku/__init__.py
  39. +5 −1 mlem/contrib/heroku/build.py
  40. +67 −53 mlem/contrib/heroku/meta.py
  41. +3 −1 mlem/contrib/heroku/server.py
  42. +3 −0 mlem/contrib/kubernetes/__init__.py
  43. +202 −0 mlem/contrib/kubernetes/base.py
  44. +30 −0 mlem/contrib/kubernetes/build.py
  45. +55 −0 mlem/contrib/kubernetes/context.py
  46. +47 −0 mlem/contrib/kubernetes/resources.yaml.j2
  47. +115 −0 mlem/contrib/kubernetes/service.py
  48. +80 −0 mlem/contrib/kubernetes/utils.py
  49. +16 −1 mlem/contrib/lightgbm.py
  50. +15 −16 mlem/contrib/numpy.py
  51. +5 −0 mlem/contrib/onnx.py
  52. +13 −5 mlem/contrib/pandas.py
  53. +6 −0 mlem/contrib/pip/__init__.py
  54. +19 −6 mlem/contrib/pip/base.py
  55. +14 −0 mlem/contrib/rabbitmq.py
  56. +5 −0 mlem/contrib/sagemaker/__init__.py
  57. +136 −0 mlem/contrib/sagemaker/build.py
  58. +12 −0 mlem/contrib/sagemaker/config.py
  59. 0 tests/pack/__init__.py → mlem/contrib/sagemaker/copy.j2
  60. +93 −0 mlem/contrib/sagemaker/env_setup.py
  61. +401 −0 mlem/contrib/sagemaker/meta.py
  62. +82 −0 mlem/contrib/sagemaker/mlem_sagemaker.tf
  63. +3 −0 mlem/contrib/sagemaker/post_copy.j2
  64. +103 −0 mlem/contrib/sagemaker/runtime.py
  65. +79 −0 mlem/contrib/sagemaker/utils.py
  66. +11 −4 mlem/contrib/sklearn.py
  67. +16 −6 mlem/contrib/tensorflow.py
  68. +19 −10 mlem/contrib/torch.py
  69. +3 −2 mlem/contrib/venv.py
  70. +12 −6 mlem/contrib/xgboost.py
  71. +8 −0 mlem/core/artifacts.py
  72. +142 −43 mlem/core/base.py
  73. +34 −30 mlem/core/data_type.py
  74. +26 −4 mlem/core/errors.py
  75. +0 −192 mlem/core/index.py
  76. +40 −13 mlem/core/meta_io.py
  77. +3 −7 mlem/core/metadata.py
  78. +13 −3 mlem/core/model.py
  79. +434 −167 mlem/core/objects.py
  80. +20 −16 mlem/core/requirements.py
  81. +21 −0 mlem/ext.py
  82. +11 −3 mlem/polydantic/core.py
  83. +4 −0 mlem/runtime/client.py
  84. +4 −0 mlem/runtime/interface.py
  85. +1 −0 mlem/ui.py
  86. +41 −11 mlem/utils/entrypoints.py
  87. +113 −0 mlem/utils/fslock.py
  88. +5 −0 mlem/utils/git.py
  89. +2 −2 mlem/utils/root.py
  90. +2 −1 mlem/utils/templates.py
  91. +1 −0 setup.cfg
  92. +19 −4 setup.py
  93. +11 −81 tests/api/test_commands.py
  94. +20 −2 tests/cli/conftest.py
  95. +33 −62 tests/cli/test_apply.py
  96. +74 −3 tests/cli/test_build.py
  97. +1 −1 tests/cli/test_clone.py
  98. +491 −1 tests/cli/test_declare.py
  99. +422 −39 tests/cli/test_deployment.py
  100. +1 −69 tests/cli/test_info.py
  101. +2 −4 tests/cli/test_init.py
  102. +3 −5 tests/cli/test_link.py
  103. +39 −23 tests/cli/test_main.py
  104. +8 −1 tests/cli/test_serve.py
  105. +8 −4 tests/cli/test_stderr.py
  106. +108 −0 tests/cli/test_types.py
  107. +24 −11 tests/conftest.py
  108. 0 tests/contrib/resources/pandas/{.mlem/config.yaml → .mlem.yaml}
  109. +16 −7 tests/contrib/test_bitbucket.py
  110. +3 −0 tests/contrib/test_docker/resources/dockerfile.j2
  111. +7 −4 tests/contrib/test_docker/test_context.py
  112. +52 −12 tests/contrib/test_docker/test_deploy.py
  113. +14 −7 tests/contrib/test_gitlab.py
  114. +8 −7 tests/contrib/test_heroku.py
  115. 0 tests/{resources/empty/.mlem/config.yaml → contrib/test_kubernetes/__init__.py}
  116. +46 −0 tests/contrib/test_kubernetes/conftest.py
  117. +126 −0 tests/contrib/test_kubernetes/test_base.py
  118. +150 −0 tests/contrib/test_kubernetes/test_context.py
  119. +34 −0 tests/contrib/test_kubernetes/utils.py
  120. +3 −4 tests/contrib/test_pandas.py
  121. +166 −3 tests/core/test_base.py
  122. +2 −2 tests/core/test_meta_io.py
  123. +7 −12 tests/core/test_metadata.py
  124. +57 −110 tests/core/test_objects.py
  125. +6 −0 tests/core/test_requirements.py
  126. 0 tests/resources/empty/.mlem.yaml
  127. 0 tests/resources/storage/{.mlem/config.yaml → .mlem.yaml}
  128. +8 −3 tests/test_config.py
  129. +69 −7 tests/test_ext.py
  130. +1 −1 tests/test_setup.py
  131. +39 −0 tests/utils/test_entrypoints.py
  132. +62 −0 tests/utils/test_fslock.py
6 changes: 5 additions & 1 deletion .github/workflows/check-test-release.yml
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ on:

env:
MLEM_TESTS: "true"
MLEM_DEBUG: "true"

jobs:
authorize:
@@ -58,7 +59,7 @@ jobs:
# no HDF5 support installed for tables
- os: windows-latest
python: "3.9"
fail-fast: true
fail-fast: false
steps:
- uses: actions/checkout@v3
with:
@@ -97,6 +98,9 @@ jobs:
pip install pre-commit .[tests]
- run: pre-commit run pylint -a -v --show-diff-on-failure
if: matrix.python != '3.7'
- name: Start minikube
if: matrix.os == 'ubuntu-latest' && matrix.python == '3.9'
uses: medyagh/setup-minikube@master
- name: Run tests
timeout-minutes: 40
run: pytest
7 changes: 4 additions & 3 deletions .pylintrc
Original file line number Diff line number Diff line change
@@ -170,7 +170,8 @@ disable=print-statement,
redefined-builtin, # TODO: https://github.com/iterative/mlem/issues/60
no-self-use, # TODO: https://github.com/iterative/mlem/issues/60 maybe leave it
import-outside-toplevel,
wrong-import-order # handeled by isort
wrong-import-order, # handeled by isort
cannot-enumerate-pytest-fixtures # TODO: https://github.com/iterative/mlem/issues/60

# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
@@ -369,7 +370,7 @@ indent-string=' '
max-line-length=100

# Maximum number of lines in a module.
max-module-lines=1000
max-module-lines=2000

# Allow the body of a class to be on the same line as the declaration if body
# contains single statement.
@@ -389,7 +390,7 @@ ignore-comments=yes
ignore-docstrings=yes

# Ignore imports when computing similarities.
ignore-imports=no
ignore-imports=yes

# Ignore function signatures when computing similarities.
ignore-signatures=no
2 changes: 0 additions & 2 deletions mlem/api/__init__.py
Original file line number Diff line number Diff line change
@@ -11,15 +11,13 @@
import_object,
init,
link,
ls,
serve,
)

__all__ = [
"save",
"load",
"load_meta",
"ls",
"clone",
"init",
"link",
123 changes: 42 additions & 81 deletions mlem/api/commands.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
MLEM's Python API
"""
import posixpath
from typing import Any, Dict, Iterable, List, Optional, Type, Union
from typing import Any, Dict, Optional, Union

from fsspec import AbstractFileSystem
from fsspec.implementations.local import LocalFileSystem
@@ -14,8 +14,7 @@
get_model_meta,
parse_import_type_modifier,
)
from mlem.config import CONFIG_FILE_NAME, project_config
from mlem.constants import PREDICT_METHOD_NAME
from mlem.constants import MLEM_CONFIG_FILE_NAME, PREDICT_METHOD_NAME
from mlem.core.errors import (
InvalidArgumentError,
MlemError,
@@ -25,7 +24,7 @@
WrongMethodError,
)
from mlem.core.import_objects import ImportAnalyzer, ImportHook
from mlem.core.meta_io import MLEM_DIR, Location, UriResolver, get_fs
from mlem.core.meta_io import Location, get_fs
from mlem.core.metadata import load_meta, save
from mlem.core.objects import (
MlemBuilder,
@@ -56,8 +55,6 @@ def apply(
method: str = None,
output: str = None,
target_project: str = None,
index: bool = None,
external: bool = None,
batch_size: Optional[int] = None,
) -> Optional[Any]:
"""Apply provided model against provided data
@@ -70,8 +67,6 @@ def apply(
If more than one is available, will fail.
output (str, optional): If value is provided,
assume it's path and save output there.
index (bool): Whether to index saved output in MLEM root folder.
external (bool): Whether to save result outside mlem dir
Returns:
If `output=None`, returns results for given data.
@@ -103,9 +98,7 @@ def apply(
return res
if len(res) == 1:
res = res[0]
return save(
res, output, project=target_project, external=external, index=index
)
return save(res, output, project=target_project)


def apply_remote(
@@ -114,7 +107,6 @@ def apply_remote(
method: str = None,
output: str = None,
target_project: str = None,
index: bool = False,
**client_kwargs,
) -> Optional[Any]:
"""Apply provided model against provided data
@@ -127,7 +119,6 @@ def apply_remote(
If more than one is available, will fail.
output (str, optional): If value is provided,
assume it's path and save output there.
index (bool): Whether to index saved output in MLEM root folder.
Returns:
If `output=None`, returns results for given data.
@@ -151,7 +142,7 @@ def apply_remote(
return res
if len(res) == 1:
res = res[0]
return save(res, output, project=target_project, index=index)
return save(res, output, project=target_project)


def clone(
@@ -164,8 +155,6 @@ def clone(
target_fs: Optional[str] = None,
follow_links: bool = True,
load_value: bool = False,
index: bool = None,
external: bool = None,
) -> MlemObject:
"""Clones MLEM object from `path` to `out`
and returns Python representation for the created object
@@ -181,8 +170,6 @@ def clone(
follow_links (bool, optional): If object we read is a MLEM link, whether to load
the actual object link points to. Defaults to True.
load_value (bool, optional): Load actual python object incorporated in MlemObject. Defaults to False.
index: whether to index object in target project
external: wheter to put object inside mlem dir in target project
Returns:
MlemObject: Copy of initial object saved to `out`
@@ -202,14 +189,12 @@ def clone(
target,
fs=target_fs,
project=target_project,
index=index,
external=external,
)


def init(path: str = ".") -> None:
"""Creates .mlem directory in `path`"""
path = posixpath.join(path, MLEM_DIR)
"""Creates mlem config in `path`"""
path = posixpath.join(path, MLEM_CONFIG_FILE_NAME)
fs, path = get_fs(path)
if fs.exists(path):
echo(
@@ -252,9 +237,8 @@ def init(path: str = ".") -> None:
"<https://mlem.ai/docs/user-guide/analytics>"
)
)
fs.makedirs(path)
# some fs dont support creating empty dirs
with fs.open(posixpath.join(path, CONFIG_FILE_NAME), "w"):
with fs.open(path, "w"):
pass
echo(
EMOJI_MLEM
@@ -273,7 +257,6 @@ def link(
rev: Optional[str] = None,
target: Optional[str] = None,
target_project: Optional[str] = None,
external: Optional[bool] = None,
follow_links: bool = True,
absolute: bool = False,
) -> MlemLink:
@@ -288,7 +271,6 @@ def link(
treat `target` as link name and dump link in MLEM DIR
follow_links (bool): Whether to make link to the underlying object
if `source` is itself a link. Defaults to True.
external (bool): Whether to save link outside mlem dir
absolute (bool): Whether to make link absolute or relative to mlem project
Returns:
@@ -308,7 +290,6 @@ def link(
return source.make_link(
target,
project=target_project,
external=external,
absolute=absolute,
)

@@ -359,25 +340,6 @@ def _validate_ls_project(loc: Location, project):
mlem_project_exists(loc.project, loc.fs, raise_on_missing=True)


def ls( # pylint: disable=too-many-locals
project: str = ".",
rev: Optional[str] = None,
fs: Optional[AbstractFileSystem] = None,
type_filter: Union[
Type[MlemObject], Iterable[Type[MlemObject]], None
] = None,
include_links: bool = True,
ignore_errors: bool = False,
) -> Dict[Type[MlemObject], List[MlemObject]]:
loc = UriResolver.resolve(
"", project=project, rev=rev, fs=fs, find_project=True
)
_validate_ls_project(loc, project)
return project_config(project, fs).index.list(
loc, type_filter, include_links, ignore_errors
)


def import_object(
path: str,
project: Optional[str] = None,
@@ -388,13 +350,11 @@ def import_object(
target_fs: Optional[AbstractFileSystem] = None,
type_: Optional[str] = None,
copy_data: bool = True,
external: bool = None,
index: bool = None,
):
"""Try to load an object as MLEM model (or data) and return it,
optionally saving to the specified target location
"""
loc = UriResolver.resolve(path, project, rev, fs)
loc = Location.resolve(path, project, rev, fs)
echo(EMOJI_LOAD + f"Importing object from {loc.uri_repr}")
if type_ is not None:
type_, modifier = parse_import_type_modifier(type_)
@@ -408,60 +368,61 @@ def import_object(
target,
fs=target_fs,
project=target_project,
index=index,
external=external,
)
return meta


def deploy(
deploy_meta_or_path: Union[MlemDeployment, str],
model: Union[MlemModel, str] = None,
model: Union[MlemModel, str],
env: Union[MlemEnv, str] = None,
project: Optional[str] = None,
rev: Optional[str] = None,
fs: Optional[AbstractFileSystem] = None,
external: bool = None,
index: bool = None,
env_kwargs: Dict[str, Any] = None,
**deploy_kwargs,
) -> MlemDeployment:
deploy_path = None
deploy_meta: MlemDeployment
update = False
if isinstance(deploy_meta_or_path, str):
deploy_path = deploy_meta_or_path
try:
deploy_meta = load_meta(
path=deploy_path,
path=deploy_meta_or_path,
project=project,
rev=rev,
fs=fs,
force_type=MlemDeployment,
)
except MlemObjectNotFound:
deploy_meta = None

update = True
except MlemObjectNotFound as e:
if env is None:
raise MlemError(
"Please provide model and env args for new deployment"
) from e
if not deploy_meta_or_path:
raise MlemError("deploy_path cannot be empty") from e

env_meta = ensure_meta(MlemEnv, env, allow_typename=True)
if isinstance(env_meta, type):
env = None
if env_kwargs:
env = env_meta(**env_kwargs)
deploy_type = env_meta.deploy_type
deploy_meta = deploy_type(
env=env,
**deploy_kwargs,
)
deploy_meta.dump(deploy_meta_or_path, fs, project)
else:
deploy_meta = deploy_meta_or_path
if model is not None:
deploy_meta.replace_model(get_model_meta(model))
update = True

if deploy_meta is None:
if model is None or env is None:
raise MlemError(
"Please provide model and env args for new deployment"
)
if not deploy_path:
raise MlemError("deploy_path cannot be empty")
model_meta = get_model_meta(model)
env_meta = ensure_meta(MlemEnv, env)
deploy_meta = env_meta.deploy_type(
model=model_meta,
env=env_meta,
env_link=env_meta.make_link(),
model_link=model_meta.make_link(),
**deploy_kwargs,
)
deploy_meta.dump(deploy_path, fs, project, index, external)
if update:
pass # todo update from deploy_args and env_args
# ensuring links are working
deploy_meta.get_env()
deploy_meta.get_model()
model_meta = get_model_meta(model)

deploy_meta.run()
deploy_meta.check_unchanged()
deploy_meta.deploy(model_meta)
return deploy_meta
Loading