diff --git a/mlem/api/__init__.py b/mlem/api/__init__.py index 7244a0d8..83c11a0f 100644 --- a/mlem/api/__init__.py +++ b/mlem/api/__init__.py @@ -11,7 +11,6 @@ import_object, init, link, - ls, serve, ) @@ -19,7 +18,6 @@ "save", "load", "load_meta", - "ls", "clone", "init", "link", diff --git a/mlem/api/commands.py b/mlem/api/commands.py index e1f00099..b0eac797 100644 --- a/mlem/api/commands.py +++ b/mlem/api/commands.py @@ -2,7 +2,7 @@ MLEM's Python API """ import posixpath -from typing import Any, Dict, Iterable, List, Optional, Type, Union +from typing import Any, Dict, Optional, Union from fsspec import AbstractFileSystem from fsspec.implementations.local import LocalFileSystem @@ -14,8 +14,7 @@ get_model_meta, parse_import_type_modifier, ) -from mlem.config import CONFIG_FILE_NAME, project_config -from mlem.constants import PREDICT_METHOD_NAME +from mlem.constants import MLEM_CONFIG_FILE_NAME, PREDICT_METHOD_NAME from mlem.core.errors import ( InvalidArgumentError, MlemError, @@ -25,7 +24,7 @@ WrongMethodError, ) from mlem.core.import_objects import ImportAnalyzer, ImportHook -from mlem.core.meta_io import MLEM_DIR, Location, get_fs +from mlem.core.meta_io import Location, get_fs from mlem.core.metadata import load_meta, save from mlem.core.objects import ( MlemBuilder, @@ -56,8 +55,6 @@ def apply( method: str = None, output: str = None, target_project: str = None, - index: bool = None, - external: bool = None, batch_size: Optional[int] = None, ) -> Optional[Any]: """Apply provided model against provided data @@ -70,8 +67,6 @@ def apply( If more than one is available, will fail. output (str, optional): If value is provided, assume it's path and save output there. - index (bool): Whether to index saved output in MLEM root folder. - external (bool): Whether to save result outside mlem dir Returns: If `output=None`, returns results for given data. @@ -103,9 +98,7 @@ def apply( return res if len(res) == 1: res = res[0] - return save( - res, output, project=target_project, external=external, index=index - ) + return save(res, output, project=target_project) def apply_remote( @@ -114,7 +107,6 @@ def apply_remote( method: str = None, output: str = None, target_project: str = None, - index: bool = False, **client_kwargs, ) -> Optional[Any]: """Apply provided model against provided data @@ -127,7 +119,6 @@ def apply_remote( If more than one is available, will fail. output (str, optional): If value is provided, assume it's path and save output there. - index (bool): Whether to index saved output in MLEM root folder. Returns: If `output=None`, returns results for given data. @@ -151,7 +142,7 @@ def apply_remote( return res if len(res) == 1: res = res[0] - return save(res, output, project=target_project, index=index) + return save(res, output, project=target_project) def clone( @@ -164,8 +155,6 @@ def clone( target_fs: Optional[str] = None, follow_links: bool = True, load_value: bool = False, - index: bool = None, - external: bool = None, ) -> MlemObject: """Clones MLEM object from `path` to `out` and returns Python representation for the created object @@ -181,8 +170,6 @@ def clone( follow_links (bool, optional): If object we read is a MLEM link, whether to load the actual object link points to. Defaults to True. load_value (bool, optional): Load actual python object incorporated in MlemObject. Defaults to False. - index: whether to index object in target project - external: wheter to put object inside mlem dir in target project Returns: MlemObject: Copy of initial object saved to `out` @@ -202,14 +189,12 @@ def clone( target, fs=target_fs, project=target_project, - index=index, - external=external, ) def init(path: str = ".") -> None: - """Creates .mlem directory in `path`""" - path = posixpath.join(path, MLEM_DIR) + """Creates mlem config in `path`""" + path = posixpath.join(path, MLEM_CONFIG_FILE_NAME) fs, path = get_fs(path) if fs.exists(path): echo( @@ -252,9 +237,8 @@ def init(path: str = ".") -> None: "" ) ) - fs.makedirs(path) # some fs dont support creating empty dirs - with fs.open(posixpath.join(path, CONFIG_FILE_NAME), "w"): + with fs.open(path, "w"): pass echo( EMOJI_MLEM @@ -273,7 +257,6 @@ def link( rev: Optional[str] = None, target: Optional[str] = None, target_project: Optional[str] = None, - external: Optional[bool] = None, follow_links: bool = True, absolute: bool = False, ) -> MlemLink: @@ -288,7 +271,6 @@ def link( treat `target` as link name and dump link in MLEM DIR follow_links (bool): Whether to make link to the underlying object if `source` is itself a link. Defaults to True. - external (bool): Whether to save link outside mlem dir absolute (bool): Whether to make link absolute or relative to mlem project Returns: @@ -308,7 +290,6 @@ def link( return source.make_link( target, project=target_project, - external=external, absolute=absolute, ) @@ -359,25 +340,6 @@ def _validate_ls_project(loc: Location, project): mlem_project_exists(loc.project, loc.fs, raise_on_missing=True) -def ls( # pylint: disable=too-many-locals - project: str = ".", - rev: Optional[str] = None, - fs: Optional[AbstractFileSystem] = None, - type_filter: Union[ - Type[MlemObject], Iterable[Type[MlemObject]], None - ] = None, - include_links: bool = True, - ignore_errors: bool = False, -) -> Dict[Type[MlemObject], List[MlemObject]]: - loc = Location.resolve( - "", project=project, rev=rev, fs=fs, find_project=True - ) - _validate_ls_project(loc, project) - return project_config(project, fs).index.list( - loc, type_filter, include_links, ignore_errors - ) - - def import_object( path: str, project: Optional[str] = None, @@ -388,8 +350,6 @@ def import_object( target_fs: Optional[AbstractFileSystem] = None, type_: Optional[str] = None, copy_data: bool = True, - external: bool = None, - index: bool = None, ): """Try to load an object as MLEM model (or data) and return it, optionally saving to the specified target location @@ -408,8 +368,6 @@ def import_object( target, fs=target_fs, project=target_project, - index=index, - external=external, ) return meta @@ -421,8 +379,6 @@ def deploy( project: Optional[str] = None, rev: Optional[str] = None, fs: Optional[AbstractFileSystem] = None, - external: bool = None, - index: bool = None, env_kwargs: Dict[str, Any] = None, **deploy_kwargs, ) -> MlemDeployment: @@ -456,7 +412,7 @@ def deploy( env=env, **deploy_kwargs, ) - deploy_meta.dump(deploy_meta_or_path, fs, project, index, external) + deploy_meta.dump(deploy_meta_or_path, fs, project) else: deploy_meta = deploy_meta_or_path update = True diff --git a/mlem/cli/__init__.py b/mlem/cli/__init__.py index 24952f95..5ea9d854 100644 --- a/mlem/cli/__init__.py +++ b/mlem/cli/__init__.py @@ -10,7 +10,7 @@ from mlem.cli.deployment import deployment from mlem.cli.dev import dev from mlem.cli.import_object import import_object -from mlem.cli.info import ls, pretty_print +from mlem.cli.info import pretty_print from mlem.cli.init import init from mlem.cli.link import link from mlem.cli.main import app @@ -25,7 +25,6 @@ "build", "pretty_print", "link", - "ls", "clone", "serve", "config", diff --git a/mlem/cli/apply.py b/mlem/cli/apply.py index d7c7635f..830e5454 100644 --- a/mlem/cli/apply.py +++ b/mlem/cli/apply.py @@ -13,9 +13,7 @@ option_data, option_data_project, option_data_rev, - option_external, option_file_conf, - option_index, option_json, option_load, option_method, @@ -80,8 +78,6 @@ def apply( import_: bool = option_import, import_type: str = option_import_type, batch_size: Optional[int] = option_batch_size, - index: bool = option_index, - external: bool = option_external, json: bool = option_json, ): """Apply a model to data. The result will be saved as a MLEM object to `output` if @@ -116,8 +112,6 @@ def apply( data, method=method, output=output, - index=index, - external=external, batch_size=batch_size, ) if output is None and json: @@ -144,7 +138,6 @@ def _apply_remote( data, project, rev, - index, method, output, target_project, @@ -169,7 +162,6 @@ def _apply_remote( data, project, rev, - index, method, output, target_project, @@ -190,7 +182,6 @@ def apply_remote_load( output: Optional[str] = option_output, target_project: Optional[str] = option_target_project, method: str = option_method, - index: bool = option_index, json: bool = option_json, load: Optional[str] = option_load("client"), ): @@ -198,7 +189,6 @@ def apply_remote_load( data, project, rev, - index, method, output, target_project, @@ -229,7 +219,6 @@ def apply_remote_func( output: Optional[str] = option_output, target_project: Optional[str] = option_target_project, method: str = option_method, - index: bool = option_index, json: bool = option_json, file_conf: List[str] = option_file_conf("client"), **__kwargs__, @@ -238,7 +227,6 @@ def apply_remote_func( data, project, rev, - index, method, output, target_project, @@ -255,7 +243,6 @@ def run_apply_remote( data_path: str, project, rev, - index, method, output, target_project, @@ -275,6 +262,5 @@ def run_apply_remote( method=method, output=output, target_project=target_project, - index=index, ) return result diff --git a/mlem/cli/clone.py b/mlem/cli/clone.py index 58c7b280..1962edd8 100644 --- a/mlem/cli/clone.py +++ b/mlem/cli/clone.py @@ -4,8 +4,6 @@ from mlem.cli.main import ( mlem_command, - option_external, - option_index, option_project, option_rev, option_target_project, @@ -19,8 +17,6 @@ def clone( project: Optional[str] = option_project, rev: Optional[str] = option_rev, target_project: Optional[str] = option_target_project, - external: Optional[bool] = option_external, - index: Optional[bool] = option_index, ): """Copy a MLEM Object from `uri` and saves a copy of it to `target` path. @@ -33,6 +29,4 @@ def clone( project=project, rev=rev, target_project=target_project, - external=external, - index=index, ) diff --git a/mlem/cli/config.py b/mlem/cli/config.py index ef8e452c..abfdc4bf 100644 --- a/mlem/cli/config.py +++ b/mlem/cli/config.py @@ -5,8 +5,8 @@ from yaml import safe_dump, safe_load from mlem.cli.main import app, mlem_command, mlem_group, option_project -from mlem.config import CONFIG_FILE_NAME, get_config_cls -from mlem.constants import MLEM_DIR +from mlem.config import get_config_cls +from mlem.constants import MLEM_CONFIG_FILE_NAME from mlem.core.base import SmartSplitDict, get_recursively, smart_split from mlem.core.errors import MlemError from mlem.core.meta_io import get_fs, get_uri @@ -41,7 +41,8 @@ def config_set( section, name = name.split(".", maxsplit=1) except ValueError as e: raise MlemError("[name] should contain at least one dot") from e - with fs.open(posixpath.join(project, MLEM_DIR, CONFIG_FILE_NAME)) as f: + config_file_path = posixpath.join(project, MLEM_CONFIG_FILE_NAME) + with fs.open(config_file_path) as f: new_conf = safe_load(f) or {} conf = SmartSplitDict(new_conf.get(section, {})) @@ -50,8 +51,7 @@ def config_set( if validate: config_cls = get_config_cls(section) config_cls(**new_conf[section]) - config_file = posixpath.join(project, MLEM_DIR, CONFIG_FILE_NAME) - with fs.open(config_file, "w", encoding="utf8") as f: + with fs.open(config_file_path, "w", encoding="utf8") as f: safe_dump( new_conf, f, @@ -73,7 +73,7 @@ def config_get( """ fs, path = get_fs(project or "") project = find_project_root(path, fs=fs) - with fs.open(posixpath.join(project, MLEM_DIR, CONFIG_FILE_NAME)) as f: + with fs.open(posixpath.join(project, MLEM_CONFIG_FILE_NAME)) as f: try: echo(get_recursively(safe_load(f), smart_split(name, "."))) except KeyError as e: diff --git a/mlem/cli/declare.py b/mlem/cli/declare.py index 6203af9f..6b7460ff 100644 --- a/mlem/cli/declare.py +++ b/mlem/cli/declare.py @@ -8,14 +8,7 @@ from ..core.meta_io import Location from ..core.objects import MlemDeployment, MlemObject from ..utils.entrypoints import list_abstractions, list_implementations -from .main import ( - app, - mlem_command, - mlem_group, - option_external, - option_index, - option_project, -) +from .main import app, mlem_command, mlem_group, option_project from .utils import ( CliTypeField, _option_from_field, @@ -99,8 +92,6 @@ def subtype_command( ..., help="Where to save the object (.mlem file)" ), project: str = option_project, - external: bool = option_external, - index: bool = option_index, **__kwargs__, ): subtype_cls = load_impl_ext(type_name, subtype) @@ -109,7 +100,7 @@ def subtype_command( meta = build_mlem_object( cls, subtype, str_conf=None, file_conf=[], **__kwargs__ ) - meta.dump(path, project=project, index=index, external=external) + meta.dump(path, project=project) for meta_type in list_implementations(MlemObject): diff --git a/mlem/cli/deployment.py b/mlem/cli/deployment.py index 929a94e4..d5335850 100644 --- a/mlem/cli/deployment.py +++ b/mlem/cli/deployment.py @@ -12,9 +12,7 @@ mlem_group_callback, option_data_project, option_data_rev, - option_external, option_file_conf, - option_index, option_json, option_load, option_method, @@ -69,8 +67,6 @@ def deploy_run_callback( model_rev: Optional[str] = option_model_rev, project: Optional[str] = option_project, rev: Optional[str] = option_rev, - external: bool = option_external, - index: bool = option_index, ): """Deploy a model to a target environment. Can use an existing deployment declaration or create a new one on-the-fly. @@ -84,8 +80,6 @@ def deploy_run_callback( ), project=project, rev=rev, - external=external, - index=index, ) @@ -111,8 +105,6 @@ def deploy_run_command( model_project: Optional[str] = option_model_project, model_rev: Optional[str] = option_model_rev, project: Optional[str] = option_project, - external: bool = option_external, - index: bool = option_index, file_conf: List[str] = option_file_conf("deployment"), **__kwargs__, ): @@ -141,8 +133,6 @@ def deploy_run_command( force_type=MlemModel, ), project=project, - external=external, - index=index, ) @@ -213,7 +203,6 @@ def deploy_apply( ), target_project: Optional[str] = option_target_project, method: str = option_method, - index: bool = option_index, json: bool = option_json, ): """Apply a deployed model to data.""" @@ -236,7 +225,6 @@ def deploy_apply( data, data_project, data_rev, - index, method, output, target_project, diff --git a/mlem/cli/import_object.py b/mlem/cli/import_object.py index 3070710b..f40e9c67 100644 --- a/mlem/cli/import_object.py +++ b/mlem/cli/import_object.py @@ -4,8 +4,6 @@ from mlem.cli.main import ( mlem_command, - option_external, - option_index, option_project, option_rev, option_target_project, @@ -26,8 +24,6 @@ def import_object( help="Whether to create a copy of file in target location or just link existing file", ), type_: Optional[str] = Option(None, "--type", help=f"Specify how to read file Available types: {list_implementations(ImportHook)}", show_default="auto infer"), # type: ignore - index: bool = option_index, - external: bool = option_external, ): """Create a `.mlem` metafile for a model or data in any file or directory.""" from mlem.api.commands import import_object @@ -40,6 +36,4 @@ def import_object( target_project=target_project, copy_data=copy, type_=type_, - external=external, - index=index, ) diff --git a/mlem/cli/info.py b/mlem/cli/info.py index 59ea4f94..9145bee8 100644 --- a/mlem/cli/info.py +++ b/mlem/cli/info.py @@ -5,9 +5,8 @@ from typer import Argument, Option from mlem.cli.main import mlem_command, option_json, option_project, option_rev -from mlem.cli.utils import Choices from mlem.core.metadata import load_meta -from mlem.core.objects import MLEM_EXT, MlemLink, MlemObject, TypedLink +from mlem.core.objects import MLEM_EXT, MlemLink, MlemObject from mlem.ui import echo, set_echo OBJECT_TYPE_NAMES = {"data": "Data"} @@ -34,74 +33,6 @@ def _print_objects_of_type(cls: Type[MlemObject], objects: List[MlemObject]): echo("", "-", meta.name, *[link] if link else []) -TYPE_ALIASES = { - "models": "model", -} - - -def _list_types(): - return [ - k - for k, v in MlemObject.non_abstract_subtypes().items() - if not issubclass(v, TypedLink) - ] - - -@mlem_command("list", aliases=["ls"], section="common") -def ls( - type_filter: Choices("all", *_list_types()) = Option( # type: ignore[valid-type] - "all", - "-t", - "--type", - help="Type of objects to list", - ), - project: str = Argument( - "", - help="Project to list from", - show_default="current directory", - metavar="project", - ), - rev: Optional[str] = option_rev, - links: bool = Option( - True, "+l/-l", "--links/--no-links", help="Whether to include links" - ), - json: bool = option_json, - ignore_errors: bool = Option( - False, "-i", "--ignore-errors", help="Ignore corrupted objects" - ), -): - """List MLEM objects inside a MLEM project.""" - from mlem.api.commands import ls - - if type_filter == "all": - types = None - else: - types = MlemObject.__type_map__[ - TYPE_ALIASES.get(type_filter, type_filter) - ] - - objects = ls( - project or ".", - rev=rev, - type_filter=types, - include_links=links, - ignore_errors=ignore_errors, - ) - if json: - print( - dumps( - { - cls.object_type: [obj.dict() for obj in objs] - for cls, objs in objects.items() - } - ) - ) - else: - for cls, objs in objects.items(): - _print_objects_of_type(cls, objs) - return {"type_filter": type_filter.value} - - @mlem_command("pprint", hidden=True) def pretty_print( path: str = Argument(..., help="Path to object"), diff --git a/mlem/cli/link.py b/mlem/cli/link.py index bec6c89d..b4eb43b1 100644 --- a/mlem/cli/link.py +++ b/mlem/cli/link.py @@ -5,7 +5,6 @@ from mlem.cli.main import ( PATH_METAVAR, mlem_command, - option_external, option_rev, option_target_project, ) @@ -26,7 +25,6 @@ def link( ), rev: Optional[str] = option_rev, target_project: Optional[str] = option_target_project, - external: bool = option_external, follow_links: bool = Option( True, "--follow-links/--no-follow-links", @@ -52,6 +50,5 @@ def link( target=target, target_project=target_project, follow_links=follow_links, - external=external or False, absolute=absolute, ) diff --git a/mlem/cli/main.py b/mlem/cli/main.py index 55aaedf5..991e44b6 100644 --- a/mlem/cli/main.py +++ b/mlem/cli/main.py @@ -31,7 +31,7 @@ _format_validation_error, get_extra_keys, ) -from mlem.constants import MLEM_DIR, PREDICT_METHOD_NAME +from mlem.constants import PREDICT_METHOD_NAME from mlem.core.errors import MlemError from mlem.telemetry import telemetry from mlem.ui import ( @@ -489,18 +489,6 @@ def inner(*iargs, **ikwargs): help="Which model method is to be applied", ) option_rev = Option(None, "--rev", help="Repo revision to use", show_default="none", metavar=COMMITISH_METAVAR) # type: ignore -option_index = Option( - None, - "--index/--no-index", - help="Whether to index output in .mlem directory", -) -option_external = Option( - None, - "--external", - "-e", - is_flag=True, - help=f"Save result not in {MLEM_DIR}, but directly in project", -) option_target_project = Option( None, "--target-project", diff --git a/mlem/config.py b/mlem/config.py index ee2fd797..691c2eff 100644 --- a/mlem/config.py +++ b/mlem/config.py @@ -10,12 +10,10 @@ from pydantic import BaseSettings, Field, parse_obj_as, root_validator from pydantic.env_settings import InitSettingsSource -from mlem.constants import MLEM_DIR +from mlem.constants import MLEM_CONFIG_FILE_NAME from mlem.core.errors import UnknownConfigSection from mlem.utils.entrypoints import MLEM_CONFIG_ENTRY_POINT, load_entrypoints -CONFIG_FILE_NAME = "config.yaml" - def _set_location_init_source(init_source: InitSettingsSource): def inner(settings: "MlemConfig"): @@ -41,7 +39,7 @@ def inner(settings: BaseSettings) -> Dict[str, Any]: project = find_project_root(config_path, fs=fs, raise_on_missing=False) if project is None: return {} - config_file = posixpath.join(project, MLEM_DIR, CONFIG_FILE_NAME) + config_file = posixpath.join(project, MLEM_CONFIG_FILE_NAME) if not fs.exists(config_file): return {} with fs.open(config_file, encoding=encoding) as f: @@ -116,8 +114,6 @@ class Config: NO_ANALYTICS: bool = False TESTS: bool = False STORAGE: Dict = {} - INDEX: Dict = {} - EXTERNAL: bool = False EMOJIS: bool = True STATE: Dict = {} SERVER: Dict = {} @@ -131,14 +127,6 @@ def storage(self): s = parse_obj_as(Storage, self.STORAGE) return s - @property - def index(self): - from mlem.core.index import Index, LinkIndex - - if not self.INDEX: - return LinkIndex() - return parse_obj_as(Index, self.INDEX) - @property def additional_extensions(self) -> List[str]: if self.ADDITIONAL_EXTENSIONS == "": diff --git a/mlem/constants.py b/mlem/constants.py index 1f09eb95..26f3f516 100644 --- a/mlem/constants.py +++ b/mlem/constants.py @@ -1,7 +1,8 @@ -MLEM_DIR = ".mlem" MLEM_STATE_DIR = ".mlem.state" MLEM_STATE_EXT = ".state" PREDICT_METHOD_NAME = "predict" PREDICT_PROBA_METHOD_NAME = "predict_proba" PREDICT_ARG_NAME = "data" + +MLEM_CONFIG_FILE_NAME = ".mlem.yaml" diff --git a/mlem/contrib/bitbucketfs.py b/mlem/contrib/bitbucketfs.py index e8209ce7..9575cbe0 100644 --- a/mlem/contrib/bitbucketfs.py +++ b/mlem/contrib/bitbucketfs.py @@ -4,7 +4,7 @@ Implementation of `BitbucketFileSystem` and `BitbucketResolver` """ import posixpath -from typing import ClassVar, List, Optional +from typing import ClassVar, Dict, Optional from urllib.parse import quote_plus, urljoin, urlparse, urlsplit import requests @@ -16,6 +16,7 @@ from mlem.config import MlemConfigBase from mlem.core.meta_io import CloudGitResolver +from mlem.utils.git import is_long_sha BITBUCKET_ORG = "https://bitbucket.org" @@ -33,6 +34,7 @@ def __init__( self.username = username self.password = password self.url = url + self.refs_cache: Dict[str, Dict[str, str]] = {} @property def auth(self): @@ -41,6 +43,7 @@ def auth(self): return None def tree(self, path: str, repo: str, rev: str): + rev = self.get_rev_sha(repo, rev) r = requests.get( urljoin( self.url, @@ -60,6 +63,7 @@ def get_default_branch(self, repo: str): return r.json()["mainbranch"]["name"] def open(self, path: str, repo: str, rev: str): + rev = self.get_rev_sha(repo, rev) r = requests.get( urljoin( self.url, @@ -70,13 +74,26 @@ def open(self, path: str, repo: str, rev: str): r.raise_for_status() return r.content - def get_refs(self, repo: str) -> List[str]: + def _get_refs(self, repo: str) -> Dict[str, str]: r = requests.get( urljoin(self.url, self.refs_endpoint.format(repo=repo)), auth=self.auth, ) r.raise_for_status() - return [v["name"] for v in r.json()["values"]] + return {v["name"]: v["target"]["hash"] for v in r.json()["values"]} + + def get_refs(self, repo: str) -> Dict[str, str]: + if repo not in self.refs_cache: + self.refs_cache[repo] = self._get_refs(repo) + return self.refs_cache[repo] + + def invalidate_cache(self): + self.refs_cache = {} + + def get_rev_sha(self, repo: str, rev: str): + if is_long_sha(rev): + return rev + return self.get_refs(repo).get(rev, rev) def check_rev(self, repo: str, rev: str) -> bool: r = requests.head( @@ -124,6 +141,7 @@ def __init__( def invalidate_cache(self, path=None): super().invalidate_cache(path) self.dircache.clear() + self.bb.invalidate_cache() def ls(self, path, detail=False, sha=None, **kwargs): path = self._strip_protocol(path) @@ -200,7 +218,7 @@ def _open( } -def ls_bb_refs(repo): +def ls_bb_refs(repo) -> Dict[str, str]: conf = BitbucketConfig.local() password = conf.PASSWORD username = conf.USERNAME diff --git a/mlem/contrib/docker/context.py b/mlem/contrib/docker/context.py index faad7f13..5a48c6dc 100644 --- a/mlem/contrib/docker/context.py +++ b/mlem/contrib/docker/context.py @@ -322,11 +322,11 @@ def write_model(self): with no_echo(): path = os.path.join(self.path, self.model_name) if self.model.is_saved: - self.model.clone(path, external=True) + self.model.clone(path) else: copy = self.model.copy() copy.model_type.bind(self.model.model_type.model) - copy.dump(path, external=True) + copy.dump(path) def write_dockerfile(self, requirements: Requirements): echo(EMOJI_BUILD + "Generating dockerfile...") diff --git a/mlem/contrib/github.py b/mlem/contrib/github.py index 097a5b16..31c7592c 100644 --- a/mlem/contrib/github.py +++ b/mlem/contrib/github.py @@ -5,7 +5,6 @@ """ import pathlib import posixpath -import re from typing import ClassVar, Dict, Optional from urllib.parse import quote_plus, urlparse @@ -14,6 +13,7 @@ from mlem.config import LOCAL_CONFIG from mlem.core.meta_io import CloudGitResolver +from mlem.utils.git import is_long_sha def ls_branches(repo_url: str) -> Dict[str, str]: @@ -59,10 +59,6 @@ def _ls_github_refs(org: str, repo: str, endpoint: str): return None -def is_long_sha(sha: str): - return re.match(r"^[a-f\d]{40}$", sha) - - class GithubResolver(CloudGitResolver): """Resolve https://github.com URLs""" diff --git a/mlem/contrib/pip/base.py b/mlem/contrib/pip/base.py index 95a4d9a1..b19ccd83 100644 --- a/mlem/contrib/pip/base.py +++ b/mlem/contrib/pip/base.py @@ -74,9 +74,7 @@ def make_distr(self, obj: MlemModel, root: str, fs: AbstractFileSystem): posixpath.join(path, "__init__.py"), fs ) with no_echo(): - obj.clone( - posixpath.join(path, "model"), fs, external=True, index=False - ) + obj.clone(posixpath.join(path, "model"), fs) with fs.open(posixpath.join(root, "requirements.txt"), "w") as f: f.write( "\n".join( diff --git a/mlem/core/errors.py b/mlem/core/errors.py index d8ff7f50..95a03652 100644 --- a/mlem/core/errors.py +++ b/mlem/core/errors.py @@ -1,7 +1,7 @@ """Exceptions raised by the MLEM.""" from typing import List, Optional -from mlem.constants import MLEM_DIR +from mlem.constants import MLEM_CONFIG_FILE_NAME class MlemError(Exception): @@ -22,7 +22,7 @@ class SerializationError(MlemError): class MlemProjectNotFound(MlemError): - _message = "{MLEM_DIR} folder wasn't found when searching through the path. Search has started from here: path={path}, fs={fs}, rev={rev}" + _message = "{MLEM_CONFIG_FILE_NAME} folder wasn't found when searching through the path. Search has started from here: path={path}, fs={fs}, rev={rev}" def __init__(self, path, fs=None, rev=None) -> None: @@ -30,7 +30,10 @@ def __init__(self, path, fs=None, rev=None) -> None: self.fs = fs self.rev = rev self.message = self._message.format( - MLEM_DIR=MLEM_DIR, path=path, fs=fs, rev=rev + MLEM_CONFIG_FILE_NAME=MLEM_CONFIG_FILE_NAME, + path=path, + fs=fs, + rev=rev, ) super().__init__(self.message) diff --git a/mlem/core/index.py b/mlem/core/index.py index 2fb41b8a..e69de29b 100644 --- a/mlem/core/index.py +++ b/mlem/core/index.py @@ -1,192 +0,0 @@ -import posixpath -from abc import abstractmethod -from collections import defaultdict -from typing import ClassVar, Dict, Iterable, List, Set, Type, Union - -from pydantic import ValidationError, parse_obj_as -from yaml import safe_dump, safe_load - -from mlem.constants import MLEM_DIR -from mlem.core.base import MlemABC -from mlem.core.errors import MlemProjectNotFound -from mlem.core.meta_io import MLEM_EXT, Location -from mlem.core.metadata import load_meta -from mlem.core.objects import MlemLink, MlemObject -from mlem.ui import no_echo - -TypeFilter = Union[Type[MlemObject], Iterable[Type[MlemObject]], None] - - -class Index(MlemABC): - """Base class for mlem object indexing logic""" - - class Config: - type_root = True - - abs_name: ClassVar = "index" - - @abstractmethod - def index(self, obj: MlemObject, location: Location): - raise NotImplementedError - - @abstractmethod - def list( - self, - location: Location, - type_filter: TypeFilter, - include_links: bool = True, - ) -> Dict[Type[MlemObject], List[MlemObject]]: - raise NotImplementedError - - @staticmethod - def parse_type_filter(type_filter: TypeFilter) -> Set[Type[MlemObject]]: - if type_filter is None: - type_filter = set(MlemObject.non_abstract_subtypes().values()) - if isinstance(type_filter, type) and issubclass( - type_filter, MlemObject - ): - type_filter = {type_filter} - tf = set(type_filter) - if not tf: - return set() - tf.add(MlemLink) - return tf - - -class LinkIndex(Index): - """Indexing base on contents of MLEM_DIR - either objects or links to them - should be there""" - - type: ClassVar = "link" - - def index(self, obj: MlemObject, location: Location): - if ( - location.path - == posixpath.join(MLEM_DIR, obj.object_type, obj.name) + MLEM_EXT - ): - return - with no_echo(): - obj.make_link( - obj.name, location.fs, project=location.project, external=False - ) - - def list( - self, - location: Location, - type_filter: TypeFilter, - include_links: bool = True, - ignore_errors: bool = False, - ) -> Dict[Type[MlemObject], List[MlemObject]]: - _type_filter = self.parse_type_filter(type_filter) - if len(_type_filter) == 0: - return {} - - res = defaultdict(list) - root_path = posixpath.join(location.project or "", MLEM_DIR) - files = location.fs.glob( - posixpath.join(root_path, f"**{MLEM_EXT}"), - ) - for cls in _type_filter: - type_path = posixpath.join(root_path, cls.object_type) - for file in files: - if not file.startswith(type_path): - continue - try: - with no_echo(): - meta = load_meta( - posixpath.relpath(file, location.project), - project=location.project, - rev=location.rev, - follow_links=False, - fs=location.fs, - load_value=False, - ) - obj_type = cls - if isinstance(meta, MlemLink): - link_name = posixpath.relpath(file, type_path)[ - : -len(MLEM_EXT) - ] - is_auto_link = meta.path == link_name + MLEM_EXT - - obj_type = MlemObject.__type_map__[meta.link_type] - if obj_type not in _type_filter: - continue - if is_auto_link: - with no_echo(): - meta = meta.load_link() - elif not include_links: - continue - res[obj_type].append(meta) - except ValidationError: - if not ignore_errors: - raise - return res - - -FileIndexSchema = Dict[str, List[str]] - - -class FileIndex(Index): - """Index as a single file""" - - type: ClassVar = "file" - filename = "index.yaml" - - def _read_index(self, location: Location): - if location.project is None: - raise MlemProjectNotFound(location.path, location.fs, location.rev) - path = posixpath.join(location.project, MLEM_DIR, self.filename) - if not location.fs.exists(path): - return {} - - with location.fs.open(path) as f: - return parse_obj_as(FileIndexSchema, safe_load(f)) - - def _write_index(self, location: Location, data: FileIndexSchema): - if location.project is None: - raise MlemProjectNotFound(location.path, location.fs, location.rev) - path = posixpath.join(location.project, MLEM_DIR, self.filename) - - with location.fs.open(path, "w") as f: - safe_dump(data, f) - - def index(self, obj: MlemObject, location: Location): - data = self._read_index(location) - type_data = data.get(obj.object_type, []) - if obj.name not in type_data: - type_data.append(obj.name) - data[obj.object_type] = type_data - self._write_index(location, data) - - def list( - self, - location: Location, - type_filter: TypeFilter, - include_links: bool = True, - ) -> Dict[Type[MlemObject], List[MlemObject]]: - _type_filter = self.parse_type_filter(type_filter) - if not _type_filter: - return {} - - data = self._read_index(location) - - res = defaultdict(list) - - with no_echo(): - for type_ in _type_filter: - if type_ is MlemLink and not include_links: - continue - - res[type_].extend( - [ - load_meta( - path, - location.project, - location.rev, - load_value=False, - fs=location.fs, - ) - for path in data.get(type_.object_type, []) - ] - ) - return res diff --git a/mlem/core/meta_io.py b/mlem/core/meta_io.py index b0c13f92..80445a57 100644 --- a/mlem/core/meta_io.py +++ b/mlem/core/meta_io.py @@ -20,7 +20,7 @@ MlemObjectNotFound, RevisionNotFound, ) -from mlem.utils.root import MLEM_DIR, find_project_root +from mlem.utils.root import find_project_root MLEM_EXT = ".mlem" @@ -293,9 +293,7 @@ def get_fs( except FileNotFoundError as e: # TODO catch HTTPError for wrong orgrepo if options["sha"] is not None and not cls.check_rev(options): raise RevisionNotFound(options["sha"], uri) from e - raise LocationNotFound( - f"Could not resolve github location {uri}" - ) from e + raise LocationNotFound(f"Could not resolve location {uri}") from e return fs, path @classmethod @@ -407,15 +405,10 @@ def get_meta_path(uri: str, fs: AbstractFileSystem) -> str: if uri.endswith(MLEM_EXT) and fs.isfile(uri): # .../. return uri - # if fs.isdir(uri) and fs.isfile(posixpath.join(uri, META_FILE_NAME)): - # # .../path and .../path/ exists - # return posixpath.join(uri, META_FILE_NAME) + if fs.isfile(uri + MLEM_EXT): # .../name without return uri + MLEM_EXT - if MLEM_DIR in uri and fs.isfile(uri): - # ...//.../file - return uri if fs.exists(uri): raise MlemObjectNotFound( f"{uri} is not a valid MLEM metafile or a folder with a MLEM model or data" diff --git a/mlem/core/metadata.py b/mlem/core/metadata.py index 5fbfc540..5562c7f4 100644 --- a/mlem/core/metadata.py +++ b/mlem/core/metadata.py @@ -47,8 +47,6 @@ def save( project: Optional[str] = None, sample_data=None, fs: Optional[AbstractFileSystem] = None, - index: bool = None, - external: Optional[bool] = None, params: Dict[str, str] = None, ) -> MlemObject: """Saves given object to a given path @@ -62,8 +60,6 @@ def save( provide input data sample, so MLEM will include it's schema in the model's metadata fs: FileSystem for the `path` argument - index: Whether to add object to mlem project index - external: if obj is saved to project, whether to put it outside of .mlem dir params: arbitrary params for object Returns: @@ -74,7 +70,7 @@ def save( sample_data, params=params, ) - meta.dump(path, fs=fs, project=project, index=index, external=external) + meta.dump(path, fs=fs, project=project) return meta diff --git a/mlem/core/objects.py b/mlem/core/objects.py index 2de57a4d..55e2bbc5 100644 --- a/mlem/core/objects.py +++ b/mlem/core/objects.py @@ -49,12 +49,11 @@ MlemError, MlemObjectNotFound, MlemObjectNotSavedError, - MlemProjectNotFound, WrongABCType, WrongMetaSubType, WrongMetaType, ) -from mlem.core.meta_io import MLEM_DIR, MLEM_EXT, Location, get_path_by_fs_path +from mlem.core.meta_io import MLEM_EXT, Location, get_path_by_fs_path from mlem.core.model import ModelAnalyzer, ModelType from mlem.core.requirements import Requirements from mlem.polydantic.lazy import lazy_field @@ -103,11 +102,7 @@ def loc(self) -> Location: @property def name(self): """Name of the object in the project""" - project_path = self.loc.path_in_project[: -len(MLEM_EXT)] - prefix = posixpath.join(MLEM_DIR, self.object_type) - if project_path.startswith(prefix): - project_path = project_path[len(prefix) + 1 :] - return project_path + return self.loc.path_in_project[: -len(MLEM_EXT)] @property def is_saved(self): @@ -134,8 +129,6 @@ def _get_location( path: str, project: Optional[str], fs: Optional[AbstractFileSystem], - external: bool, - ensure_mlem_root: bool, metafile_path: bool = True, ) -> Location: """Create location from arguments""" @@ -149,24 +142,6 @@ def _get_location( find_project_root( loc.project, loc.fs, raise_on_missing=True, recursive=False ) - if ensure_mlem_root and loc.project is None: - raise MlemProjectNotFound(loc.fullpath, loc.fs) - if ( - loc.project is None - or external - or loc.fullpath.startswith( - posixpath.join(loc.project, MLEM_DIR, cls.object_type) - ) - ): - # orphan or external or inside .mlem - return loc - - internal_path = posixpath.join( - MLEM_DIR, - cls.object_type, - loc.path_in_project, - ) - loc.update_path(internal_path) return loc @classmethod @@ -219,8 +194,6 @@ def dump( path: str, fs: Optional[AbstractFileSystem] = None, project: Optional[str] = None, - index: Optional[bool] = None, - external: Optional[bool] = None, ): """Dumps metafile and possible artifacts to path. @@ -228,23 +201,14 @@ def dump( path: name of the object. Relative to project, if it is provided. fs: filesystem to save to. if not provided, inferred from project and path project: path to mlem project - index: whether add to index if object is external. - If set to True, checks existanse of mlem project - defaults to True if mlem project exists and external is true - external: whether to save object inside mlem dir or not. - Defaults to false if project is provided - Forced to false if path points inside mlem dir """ - location, index = self._parse_dump_args( - path, project, fs, index, external - ) - self._write_meta(location, index) + location = self._parse_dump_args(path, project, fs) + self._write_meta(location) return self def _write_meta( self, location: Location, - index: bool, ): """Write metadata to path in fs and possibly create link in mlem dir""" echo(EMOJI_SAVE + f"Saving {self.object_type} to {location.uri_repr}") @@ -253,50 +217,27 @@ def _write_meta( ) with location.open("w") as f: safe_dump(self.dict(), f) - if index and location.project: - project_config(location.project, location.fs).index.index( - self, location - ) def _parse_dump_args( self, path: str, project: Optional[str], fs: Optional[AbstractFileSystem], - index: Optional[bool], - external: Optional[bool], - ) -> Tuple[Location, bool]: + ) -> Location: """Parse arguments for .dump and bind meta""" - if external is None: - external = project_config(project, fs=fs).EXTERNAL - # by default we index only external non-orphan objects - if index is None: - index = True - ensure_mlem_root = False - else: - # if index manually set to True, there should be mlem project - ensure_mlem_root = index location = self._get_location( make_posix(path), make_posix(project), fs, - external, - ensure_mlem_root, ) self.bind(location) - if location.project is not None: - # force external=False if fullpath inside MLEM_DIR - external = posixpath.join(MLEM_DIR, "") not in posixpath.dirname( - location.fullpath - ) - return location, index + return location def make_link( self, path: str = None, fs: Optional[AbstractFileSystem] = None, project: Optional[str] = None, - external: Optional[bool] = None, absolute: bool = False, ) -> "MlemLink": if self.location is None: @@ -310,11 +251,10 @@ def make_link( link_type=self.resolved_type, ) if path is not None: - ( - location, - _, - ) = link._parse_dump_args( # pylint: disable=protected-access - path, project, fs, False, external=external + location = ( + link._parse_dump_args( # pylint: disable=protected-access + path, project, fs + ) ) if ( not absolute @@ -324,9 +264,7 @@ def make_link( link.path = self.get_metafile_path(self.name) link.link_type = self.resolved_type link.project = None - link._write_meta( # pylint: disable=protected-access - location, True - ) + link._write_meta(location) # pylint: disable=protected-access return link def clone( @@ -334,8 +272,6 @@ def clone( path: str, fs: Optional[AbstractFileSystem] = None, project: Optional[str] = None, - index: Optional[bool] = None, - external: Optional[bool] = None, ): """ Clone existing object to `path`. @@ -346,7 +282,7 @@ def clone( raise MlemObjectNotSavedError("Cannot clone not saved object") new: "MlemObject" = self.deepcopy() new.dump( - path, fs, project, index, external + path, fs, project ) # only dump meta TODO: https://github.com/iterative/mlem/issues/37 return new @@ -363,7 +299,7 @@ def update(self): + f"Updating {self.object_type} at {self.location.uri_repr}" ) with no_echo(): - self._write_meta(self.location, False) + self._write_meta(self.location) def meta_hash(self): return hashlib.md5(safe_dump(self.dict()).encode("utf8")).hexdigest() @@ -543,9 +479,6 @@ def get_metafile_path(cls, fullpath: str): @property def name(self): project_path = self.location.path_in_project - prefix = posixpath.join(MLEM_DIR, self.object_type) - if project_path.startswith(prefix): - project_path = project_path[len(prefix) + 1 :] if project_path.endswith(MLEM_EXT): project_path = project_path[: -len(MLEM_EXT)] return project_path @@ -569,12 +502,8 @@ def dump( path: str, fs: Optional[AbstractFileSystem] = None, project: Optional[str] = None, - index: Optional[bool] = None, - external: Optional[bool] = None, ): - location, index = self._parse_dump_args( - path, project, fs, index, external - ) + location = self._parse_dump_args(path, project, fs) try: if location.exists(): with no_echo(): @@ -585,35 +514,27 @@ def dump( except (MlemObjectNotFound, FileNotFoundError, ValidationError): pass self.artifacts = self.get_artifacts() - self._write_meta(location, index) + self._write_meta(location) return self @abstractmethod def write_value(self) -> Artifacts: raise NotImplementedError - # def ensure_saved(self): - # if self.fs is None: - # raise ValueError(f"Can't load {self}: it's not saved") - def clone( self, path: str, fs: Optional[AbstractFileSystem] = None, project: Optional[str] = None, - index: Optional[bool] = None, - external: Optional[bool] = None, ): if self.location is None: raise MlemObjectNotSavedError("Cannot clone not saved object") # clone is just dump with copying artifacts new: "_WithArtifacts" = self.deepcopy() new.artifacts = {} - ( - location, - index, - ) = new._parse_dump_args( # pylint: disable=protected-access - path, project, fs, index, external + + location = new._parse_dump_args( # pylint: disable=protected-access + path, project, fs ) for art_name, art in (self.artifacts or {}).items(): @@ -627,7 +548,7 @@ def clone( new.artifacts[art_name] = LocalArtifact( uri=posixpath.relpath(art_path, new.dirname), **download.info ) - new._write_meta(location, index) # pylint: disable=protected-access + new._write_meta(location) # pylint: disable=protected-access return new @property @@ -1235,8 +1156,6 @@ def find_object( tp, posixpath.join( project or "", - MLEM_DIR, - cls.object_type, cls.get_metafile_path(path), ), ) diff --git a/mlem/utils/git.py b/mlem/utils/git.py new file mode 100644 index 00000000..d288a84b --- /dev/null +++ b/mlem/utils/git.py @@ -0,0 +1,5 @@ +import re + + +def is_long_sha(sha: str): + return re.match(r"^[a-f\d]{40}$", sha) diff --git a/mlem/utils/root.py b/mlem/utils/root.py index 3dc96e76..858dd5ef 100644 --- a/mlem/utils/root.py +++ b/mlem/utils/root.py @@ -6,7 +6,7 @@ from fsspec.implementations.local import LocalFileSystem from typing_extensions import Literal -from mlem.constants import MLEM_DIR +from mlem.constants import MLEM_CONFIG_FILE_NAME from mlem.core.errors import MlemProjectNotFound @@ -15,7 +15,7 @@ def mlem_project_exists( ): """Check is mlem project exists at path""" try: - exists = fs.exists(posixpath.join(path, MLEM_DIR)) + exists = fs.exists(posixpath.join(path, MLEM_CONFIG_FILE_NAME)) except ValueError: # some fsspec implementations throw ValueError because of # wrong bucket/container names containing "." diff --git a/tests/api/test_commands.py b/tests/api/test_commands.py index 3f089916..ef2aa03e 100644 --- a/tests/api/test_commands.py +++ b/tests/api/test_commands.py @@ -8,19 +8,15 @@ from pytest_lazyfixture import lazy_fixture from mlem.api import apply, apply_remote, link, load_meta -from mlem.api.commands import build, import_object, init, ls -from mlem.config import CONFIG_FILE_NAME -from mlem.constants import PREDICT_METHOD_NAME -from mlem.contrib.heroku.meta import HerokuEnv +from mlem.api.commands import build, import_object, init +from mlem.constants import MLEM_CONFIG_FILE_NAME, PREDICT_METHOD_NAME from mlem.core.artifacts import LocalArtifact -from mlem.core.errors import MlemProjectNotFound -from mlem.core.meta_io import MLEM_DIR, MLEM_EXT +from mlem.core.meta_io import MLEM_EXT from mlem.core.metadata import load from mlem.core.model import ModelIO -from mlem.core.objects import MlemData, MlemEnv, MlemLink, MlemModel +from mlem.core.objects import MlemLink, MlemModel from mlem.runtime.client import HTTPClient -from mlem.utils.path import make_posix -from tests.conftest import MLEM_TEST_REPO, long, need_test_repo_auth +from tests.conftest import MLEM_TEST_REPO, long IMPORT_MODEL_FILENAME = "mymodel" @@ -62,7 +58,7 @@ def test_apply_remote(mlem_client, train): def test_link_as_separate_file(model_path_mlem_project): model_path, mlem_project = model_path_mlem_project link_path = os.path.join(mlem_project, "latest.mlem") - link(model_path, target=link_path, external=True) + link(model_path, target=link_path) assert os.path.exists(link_path) link_object = load_meta(link_path, follow_links=False) assert isinstance(link_object, MlemLink) @@ -77,12 +73,9 @@ def test_link_in_mlem_dir(model_path_mlem_project): model_path, target=link_name, target_project=mlem_project, - external=False, ) assert isinstance(link_obj, MlemLink) - link_dumped_to = os.path.join( - mlem_project, MLEM_DIR, "link", link_name + MLEM_EXT - ) + link_dumped_to = os.path.join(mlem_project, link_name + MLEM_EXT) assert os.path.exists(link_dumped_to) loaded_link_object = load_meta(link_dumped_to, follow_links=False) assert isinstance(loaded_link_object, MlemLink) @@ -116,77 +109,16 @@ def test_link_from_remote_to_local(current_test_branch, mlem_project): assert isinstance(model, MlemModel) -def test_ls_local(filled_mlem_project): - objects = ls(filled_mlem_project) - assert len(objects) == 1 - assert MlemModel in objects - models = objects[MlemModel] - assert len(models) == 2 - model, lnk = models - if isinstance(model, MlemLink): - model, lnk = lnk, model - - assert isinstance(model, MlemModel) - assert isinstance(lnk, MlemLink) - assert ( - posixpath.join(make_posix(filled_mlem_project), lnk.path) - == model.loc.fullpath - ) - - -def test_ls_no_project(tmpdir): - with pytest.raises(MlemProjectNotFound): - ls(str(tmpdir)) - - -@long -@need_test_repo_auth -def test_ls_remote(current_test_branch): - objects = ls( - os.path.join(MLEM_TEST_REPO, f"tree/{current_test_branch}/simple") - ) - assert len(objects) == 2 - assert MlemModel in objects - models = objects[MlemModel] - assert len(models) == 2 - model, lnk = models - if isinstance(model, MlemLink): - model, lnk = lnk, model - - assert isinstance(model, MlemModel) - assert isinstance(lnk, MlemLink) - - assert MlemData in objects - assert len(objects[MlemData]) == 4 - - -@long -def test_ls_remote_s3(s3_tmp_path): - path = s3_tmp_path("ls_remote_s3") - init(path) - meta = HerokuEnv() - meta.dump(posixpath.join(path, "env")) - meta.dump(posixpath.join(path, "subdir", "env")) - meta.dump(posixpath.join(path, "subdir", "subsubdir", "env")) - objects = ls(path) - assert MlemEnv in objects - envs = objects[MlemEnv] - assert len(envs) == 3 - assert all(o == meta for o in envs) - - def test_init(tmpdir): init(str(tmpdir)) - assert os.path.isdir(tmpdir / MLEM_DIR) - assert os.path.isfile(tmpdir / MLEM_DIR / CONFIG_FILE_NAME) + assert os.path.isfile(tmpdir / MLEM_CONFIG_FILE_NAME) @long def test_init_remote(s3_tmp_path, s3_storage_fs): path = s3_tmp_path("init") init(path) - assert s3_storage_fs.isdir(f"{path}/{MLEM_DIR}") - assert s3_storage_fs.isfile(f"{path}/{MLEM_DIR}/{CONFIG_FILE_NAME}") + assert s3_storage_fs.isfile(f"{path}/{MLEM_CONFIG_FILE_NAME}") def _check_meta(meta, out_path, fs=None): @@ -262,9 +194,7 @@ def test_import_model_pickle__no_copy_in_mlem_project( write_model_pickle(path) out_path = os.path.join(mlem_project, "mlem_model") - meta = import_object( - path, target=out_path, type_=type_, copy_data=False, external=True - ) + meta = import_object(path, target=out_path, type_=type_, copy_data=False) _check_meta(meta, out_path) _check_load_artifact(meta, out_path, False, train, filename) @@ -297,7 +227,7 @@ def test_import_model_pickle_remote_in_project( write_model_pickle(path, s3_storage_fs) out_path = posixpath.join(project_path, "mlem_model") meta = import_object( - path, target=out_path, copy_data=False, type_="pickle", external=True + path, target=out_path, copy_data=False, type_="pickle" ) _check_meta(meta, out_path, s3_storage_fs) _check_load_artifact(meta, out_path, False, train) diff --git a/tests/cli/test_apply.py b/tests/cli/test_apply.py index e7c0f0f4..7212cce9 100644 --- a/tests/cli/test_apply.py +++ b/tests/cli/test_apply.py @@ -10,7 +10,7 @@ from mlem.api import load, save from mlem.core.data_type import ArrayType -from mlem.core.errors import MlemProjectNotFound, UnsupportedDataBatchLoading +from mlem.core.errors import UnsupportedDataBatchLoading from mlem.core.metadata import load_meta from mlem.core.objects import MlemData from mlem.runtime.client import HTTPClient @@ -36,7 +36,6 @@ def test_apply(runner, model_path, data_path): "predict", "-o", path, - "--no-index", ], ) assert result.exit_code == 0, ( @@ -61,14 +60,14 @@ def model_train_batch(): def model_path_batch(model_train_batch, tmp_path_factory): path = os.path.join(tmp_path_factory.getbasetemp(), "saved-model") model, train = model_train_batch - save(model, path, sample_data=train, index=False) + save(model, path, sample_data=train) yield path @pytest.fixture def data_path_batch(model_train_batch, tmpdir_factory): temp_dir = str(tmpdir_factory.mktemp("saved-data") / "data") - save(model_train_batch[1], temp_dir, index=False) + save(model_train_batch[1], temp_dir) yield temp_dir @@ -85,7 +84,6 @@ def test_apply_batch(runner, model_path_batch, data_path_batch): "predict", "-o", path, - "--no-index", "-b", "5", ], @@ -119,7 +117,6 @@ def test_apply_with_import(runner, model_meta_saved_single, tmp_path_factory): "predict", "-o", path, - "--no-index", "--import", "--it", "pandas[csv]", @@ -155,7 +152,6 @@ def test_apply_batch_with_import( "predict", "-o", path, - "--no-index", "--import", "--it", "pandas[csv]", @@ -168,7 +164,7 @@ def test_apply_batch_with_import( def test_apply_no_output(runner, model_path, data_path): result = runner.invoke( - ["apply", model_path, data_path, "-m", "predict", "--no-index"], + ["apply", model_path, data_path, "-m", "predict"], ) assert result.exit_code == 0, ( result.stdout, @@ -178,29 +174,6 @@ def test_apply_no_output(runner, model_path, data_path): assert len(result.stdout) > 0 -def test_apply_fails_without_mlem_dir(runner, model_path, data_path): - with tempfile.TemporaryDirectory() as dir: - result = runner.invoke( - [ - "--tb", - "apply", - model_path, - data_path, - "-m", - "predict", - "-o", - dir, - "--index", - ], - ) - assert result.exit_code == 1, ( - result.stdout, - result.stderr, - result.exception, - ) - assert isinstance(result.exception, MlemProjectNotFound) - - @long @need_test_repo_auth def test_apply_from_remote(runner, current_test_branch, s3_tmp_path): @@ -224,7 +197,6 @@ def test_apply_from_remote(runner, current_test_branch, s3_tmp_path): current_test_branch, "-o", out, - "--no-index", ], ) assert result.exit_code == 0, ( diff --git a/tests/cli/test_clone.py b/tests/cli/test_clone.py index 161997f8..8cb20183 100644 --- a/tests/cli/test_clone.py +++ b/tests/cli/test_clone.py @@ -8,7 +8,7 @@ def test_model_cloning(runner: Runner, model_path): with tempfile.TemporaryDirectory() as path: path = posixpath.join(path, "cloned") - result = runner.invoke(["clone", model_path, path, "--no-index"]) + result = runner.invoke(["clone", model_path, path]) assert result.exit_code == 0, ( result.stdout, result.stderr, diff --git a/tests/cli/test_declare.py b/tests/cli/test_declare.py index 1886eaa2..90a099be 100644 --- a/tests/cli/test_declare.py +++ b/tests/cli/test_declare.py @@ -281,13 +281,13 @@ class MaskedField(_MockBuilder): """mock""" field: ListValue - index: str + project: str all_test_params.append( pytest.param( - MaskedField(index="a", field=ListValue(f=["a"])), - "--.index a --field.f.0 a", + MaskedField(project="a", field=ListValue(f=["a"])), + "--.project a --field.f.0 a", id="masked", ) ) diff --git a/tests/cli/test_info.py b/tests/cli/test_info.py index c2a3c415..a74e9f76 100644 --- a/tests/cli/test_info.py +++ b/tests/cli/test_info.py @@ -1,80 +1,12 @@ import json import os -import pytest from pydantic import parse_obj_as from mlem.core.meta_io import MLEM_EXT -from mlem.core.objects import MlemLink, MlemModel, MlemObject +from mlem.core.objects import MlemModel, MlemObject from tests.conftest import MLEM_TEST_REPO, long -LOCAL_LS_EXPECTED_RESULT = """Models: - - latest -> model1 - - model1 -""" - - -@pytest.mark.parametrize("obj_type", [None, "all", "model"]) -def test_ls(runner, filled_mlem_project, obj_type): - os.chdir(filled_mlem_project) - result = runner.invoke( - ["list", "-t", obj_type] if obj_type else ["list"], - ) - assert result.exit_code == 0, ( - result.stdout, - result.stderr, - result.exception, - ) - assert len(result.stdout) > 0, "Output is empty, but should not be" - assert result.stdout == LOCAL_LS_EXPECTED_RESULT - - result = runner.invoke( - (["list", "-t", obj_type] if obj_type else ["list"]) + ["--json"], - ) - assert result.exit_code == 0, ( - result.stdout, - result.stderr, - result.exception, - ) - assert len(result.stdout) > 0, "Output is empty, but should not be" - data = json.loads(result.stdout) - assert "model" in data - models = data["model"] - assert len(models) == 2 - model, link = [parse_obj_as(MlemObject, m) for m in models] - if isinstance(model, MlemLink): - model, link = link, model - assert isinstance(model, MlemModel) - assert isinstance(link, MlemLink) - - -REMOTE_LS_EXPECTED_RESULT = """Models: - - data/model - - latest -> data/model -Data: - - data/pred - - data/test_x - - data/test_y - - data/train -""" - - -@pytest.mark.long -def test_ls_remote(runner, current_test_branch): - result = runner.invoke( - [ - "list", - f"{MLEM_TEST_REPO}/tree/{current_test_branch}/simple", - ], - ) - assert result.exit_code == 0, ( - result.stdout, - result.stderr, - result.exception, - ) - assert len(result.stdout) > 0, "Output is empty, but should not be" - assert result.stdout == REMOTE_LS_EXPECTED_RESULT - def test_pretty_print(runner, model_path_mlem_project): model_path, _ = model_path_mlem_project diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 65d08efb..1cd77053 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -1,7 +1,6 @@ import os -from mlem.config import CONFIG_FILE_NAME -from mlem.constants import MLEM_DIR +from mlem.constants import MLEM_CONFIG_FILE_NAME from mlem.utils.path import make_posix from tests.cli.conftest import Runner @@ -9,5 +8,4 @@ def test_init(runner: Runner, tmpdir): result = runner.invoke(f"init {make_posix(str(tmpdir))}") assert result.exit_code == 0, result.exception - assert os.path.isdir(tmpdir / MLEM_DIR) - assert os.path.isfile(tmpdir / MLEM_DIR / CONFIG_FILE_NAME) + assert os.path.isfile(tmpdir / MLEM_CONFIG_FILE_NAME) diff --git a/tests/cli/test_link.py b/tests/cli/test_link.py index 831f3979..4549d964 100644 --- a/tests/cli/test_link.py +++ b/tests/cli/test_link.py @@ -2,7 +2,7 @@ import tempfile from mlem.api import load_meta -from mlem.core.meta_io import MLEM_DIR, MLEM_EXT +from mlem.core.meta_io import MLEM_EXT from mlem.core.objects import MlemLink, MlemModel @@ -10,7 +10,7 @@ def test_link(runner, model_path): with tempfile.TemporaryDirectory() as dir: link_path = os.path.join(dir, "latest.mlem") result = runner.invoke( - ["link", model_path, link_path, "-e", "--abs"], + ["link", model_path, link_path, "--abs"], ) assert result.exit_code == 0, ( result.stdout, @@ -33,9 +33,7 @@ def test_link_mlem_project(runner, model_path_mlem_project): result.stderr, result.exception, ) - link_path = os.path.join( - project, MLEM_DIR, MlemLink.object_type, link_name - ) + link_path = os.path.join(project, link_name) assert os.path.exists(link_path) link_object = load_meta(link_path, follow_links=False) assert isinstance(link_object, MlemLink) diff --git a/tests/cli/test_stderr.py b/tests/cli/test_stderr.py index db8a65e2..6caf80dc 100644 --- a/tests/cli/test_stderr.py +++ b/tests/cli/test_stderr.py @@ -13,10 +13,10 @@ def test_stderr_exception(runner): # patch the ls command and ensure it throws an expection. with mock.patch( - "mlem.api.commands.ls", side_effect=Exception(EXCEPTION_MESSAGE) + "mlem.api.commands.init", side_effect=Exception(EXCEPTION_MESSAGE) ): result = runner.invoke( - ["list"], + ["init"], ) assert result.exit_code == 1, ( result.stdout, @@ -34,10 +34,10 @@ def test_stderr_exception(runner): def test_stderr_mlem_error(runner): # patch the ls command and ensure it throws a mlem error. with mock.patch( - "mlem.api.commands.ls", side_effect=MlemError(MLEM_ERROR_MESSAGE) + "mlem.api.commands.init", side_effect=MlemError(MLEM_ERROR_MESSAGE) ): result = runner.invoke( - ["list"], + ["init"], ) assert result.exit_code == 1, ( result.stdout, diff --git a/tests/conftest.py b/tests/conftest.py index fb848149..0e42fda6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ import posixpath import tempfile from pathlib import Path -from typing import Any, Callable, Type +from typing import Any, Callable, Set, Type import git import numpy as np @@ -76,8 +76,7 @@ def _check_github_test_repo_auth(): ) -@pytest.fixture() -def current_test_branch(): +def get_current_test_branch(branch_list: Set[str]): try: branch = Repo(str(Path(__file__).parent.parent)).active_branch.name except TypeError: @@ -86,14 +85,18 @@ def current_test_branch(): branch = os.environ.get("GITHUB_HEAD_REF", os.environ["GITHUB_REF"]) if branch.startswith("refs/heads/"): branch = branch[len("refs/heads/") :] - remote_refs = set( - ls_github_branches(MLEM_TEST_REPO_ORG, MLEM_TEST_REPO_NAME).keys() - ) - if branch in remote_refs: + if branch in branch_list: return branch return "main" +@pytest.fixture() +def current_test_branch(): + return get_current_test_branch( + set(ls_github_branches(MLEM_TEST_REPO_ORG, MLEM_TEST_REPO_NAME).keys()) + ) + + @pytest.fixture(scope="session", autouse=True) def add_test_env(): os.environ["MLEM_TESTS"] = "true" @@ -211,14 +214,14 @@ def model_path(model_train_target, tmp_path_factory): model, train, _ = model_train_target # because of index=False we test reading by path here # reading by link name is not tested - save(model, path, sample_data=train, index=False) + save(model, path, sample_data=train) yield path @pytest.fixture def data_path(train, tmpdir_factory): temp_dir = str(tmpdir_factory.mktemp("saved-data") / "data") - save(train, temp_dir, index=False) + save(train, temp_dir) yield temp_dir @@ -295,7 +298,7 @@ def filled_mlem_project(mlem_project): requirements=Requirements.new("sklearn"), model_type=SklearnModel(methods={}, model=""), ) - model.dump("model1", project=mlem_project, external=True) + model.dump("model1", project=mlem_project) model.make_link("latest", project=mlem_project) yield mlem_project @@ -307,7 +310,7 @@ def model_path_mlem_project(model_train_target, tmpdir_factory): dir = str(tmpdir_factory.mktemp("mlem-root-with-model")) init(dir) model_dir = os.path.join(dir, "generated-model") - save(model, model_dir, sample_data=train, index=True, external=True) + save(model, model_dir, sample_data=train) yield model_dir, dir diff --git a/tests/contrib/resources/pandas/.mlem/config.yaml b/tests/contrib/resources/pandas/.mlem.yaml similarity index 100% rename from tests/contrib/resources/pandas/.mlem/config.yaml rename to tests/contrib/resources/pandas/.mlem.yaml diff --git a/tests/contrib/test_bitbucket.py b/tests/contrib/test_bitbucket.py index 5f018803..8c5f522c 100644 --- a/tests/contrib/test_bitbucket.py +++ b/tests/contrib/test_bitbucket.py @@ -3,12 +3,12 @@ import pytest from pytest_lazyfixture import lazy_fixture -from mlem.contrib.bitbucketfs import BitBucketFileSystem +from mlem.contrib.bitbucketfs import BitBucketFileSystem, ls_bb_refs from mlem.core.errors import RevisionNotFound from mlem.core.meta_io import Location, get_fs from mlem.core.metadata import load_meta from mlem.core.objects import MlemModel -from tests.conftest import long +from tests.conftest import get_current_test_branch, long MLEM_TEST_REPO_PROJECT = "iterative-ai/mlem-test" @@ -31,6 +31,11 @@ def fs_auth(): return BitBucketFileSystem(MLEM_TEST_REPO_PROJECT) +@pytest.fixture() +def current_test_branch_bb(): + return get_current_test_branch(set(ls_bb_refs(MLEM_TEST_REPO_PROJECT))) + + @long @pytest.mark.parametrize( "fs", @@ -86,6 +91,10 @@ def test_uri_resolver_wrong_rev(): @long -def test_loading_object(): - meta = load_meta("latest", project=MLEM_TEST_REPO_URI + "/src/main/simple") +def test_loading_object(current_test_branch_bb): + meta = load_meta( + "latest", + project=MLEM_TEST_REPO_URI + "/src/main/simple", + rev=current_test_branch_bb, + ) assert isinstance(meta, MlemModel) diff --git a/tests/contrib/test_gitlab.py b/tests/contrib/test_gitlab.py index ac9a409a..5ee4822b 100644 --- a/tests/contrib/test_gitlab.py +++ b/tests/contrib/test_gitlab.py @@ -1,17 +1,22 @@ import pytest -from mlem.contrib.gitlabfs import GitlabFileSystem +from mlem.contrib.gitlabfs import GitlabFileSystem, ls_gitlab_refs from mlem.core.errors import RevisionNotFound from mlem.core.meta_io import Location, get_fs from mlem.core.metadata import load_meta from mlem.core.objects import MlemModel -from tests.conftest import long +from tests.conftest import get_current_test_branch, long MLEM_TEST_REPO_PROJECT = "iterative.ai/mlem-test" MLEM_TEST_REPO_URI = f"https://gitlab.com/{MLEM_TEST_REPO_PROJECT}" +@pytest.fixture() +def current_test_branch_gl(): + return get_current_test_branch(set(ls_gitlab_refs(MLEM_TEST_REPO_PROJECT))) + + @long def test_ls(): fs = GitlabFileSystem(MLEM_TEST_REPO_PROJECT) @@ -61,8 +66,10 @@ def test_uri_resolver_wrong_rev(): @long -def test_loading_object(): +def test_loading_object(current_test_branch_gl): meta = load_meta( - "latest", project=MLEM_TEST_REPO_URI + "/-/blob/main/simple" + "latest", + project=MLEM_TEST_REPO_URI + "/-/blob/main/simple", + rev=current_test_branch_gl, ) assert isinstance(meta, MlemModel) diff --git a/tests/contrib/test_pandas.py b/tests/contrib/test_pandas.py index d8c258cd..d7832008 100644 --- a/tests/contrib/test_pandas.py +++ b/tests/contrib/test_pandas.py @@ -14,8 +14,7 @@ from sklearn.model_selection import train_test_split from mlem.api.commands import import_object -from mlem.config import CONFIG_FILE_NAME -from mlem.constants import MLEM_DIR +from mlem.constants import MLEM_CONFIG_FILE_NAME from mlem.contrib.pandas import ( PANDAS_FORMATS, PANDAS_SERIES_FORMATS, @@ -466,7 +465,7 @@ def iris_data(): def test_save_load(iris_data, tmpdir): tmpdir = str(tmpdir / "data") - save(iris_data, tmpdir, index=False) + save(iris_data, tmpdir) data2 = load(tmpdir) pandas_assert(data2, iris_data) @@ -596,7 +595,7 @@ def test_series(series_data2: pd.Series, series_df_type2, df_type2): def test_change_format(mlem_project, data): with open( - os.path.join(mlem_project, MLEM_DIR, CONFIG_FILE_NAME), + os.path.join(mlem_project, MLEM_CONFIG_FILE_NAME), "w", encoding="utf8", ) as f: diff --git a/tests/core/test_metadata.py b/tests/core/test_metadata.py index f4b85231..89f4a03c 100644 --- a/tests/core/test_metadata.py +++ b/tests/core/test_metadata.py @@ -11,10 +11,9 @@ from sklearn.tree import DecisionTreeClassifier from mlem.api import init -from mlem.constants import MLEM_DIR from mlem.core.meta_io import MLEM_EXT from mlem.core.metadata import load, load_meta, save -from mlem.core.objects import MlemLink, MlemModel +from mlem.core.objects import MlemModel from tests.conftest import ( MLEM_TEST_REPO, MLEM_TEST_REPO_NAME, @@ -44,7 +43,7 @@ def test_model_saving_without_sample_data(model, tmpdir_factory): tmpdir_factory.mktemp("saving-models-without-sample-data") / "model" ) # index=True would require having .mlem folder somewhere - save(model, path, index=False) + save(model, path) def test_model_saving_in_mlem_project_root(model_train_target, tmpdir_factory): @@ -52,7 +51,7 @@ def test_model_saving_in_mlem_project_root(model_train_target, tmpdir_factory): init(project) model_dir = os.path.join(project, "generated-model") model, train, _ = model_train_target - save(model, model_dir, sample_data=train, index=True) + save(model, model_dir, sample_data=train) def test_model_saving(model_path): @@ -93,8 +92,7 @@ def test_meta_loading(model_path): [ f"github://{MLEM_TEST_REPO_ORG}:{MLEM_TEST_REPO_NAME}@{{branch}}/simple/data/model", f"github://{MLEM_TEST_REPO_ORG}:{MLEM_TEST_REPO_NAME}@{{branch}}/simple/data/model.mlem", - f"github://{MLEM_TEST_REPO_ORG}:{MLEM_TEST_REPO_NAME}@{{branch}}/simple/.mlem/link/data/model.mlem", - f"github://{MLEM_TEST_REPO_ORG}:{MLEM_TEST_REPO_NAME}@{{branch}}/simple/.mlem/link/latest.mlem", + f"github://{MLEM_TEST_REPO_ORG}:{MLEM_TEST_REPO_NAME}@{{branch}}/simple/latest.mlem", f"{MLEM_TEST_REPO}tree/{{branch}}/simple/data/model/", ], ) @@ -112,8 +110,7 @@ def test_model_loading_from_github_with_fsspec(url, current_test_branch): [ "data/model", "data/model.mlem", - ".mlem/link/data/model.mlem", - ".mlem/link/latest.mlem", + "latest.mlem", ], ) def test_model_loading_from_github(path, current_test_branch): @@ -149,11 +146,9 @@ def test_saving_to_s3(model, s3_storage_fs, s3_tmp_path): path = s3_tmp_path("model_save") init(path) model_path = posixpath.join(path, "model") - save(model, model_path, fs=s3_storage_fs, external=True) + save(model, model_path, fs=s3_storage_fs) model_path = model_path[len("s3:/") :] - assert s3_storage_fs.isfile( - posixpath.join(path, MLEM_DIR, MlemLink.object_type, "model.mlem") - ) + assert s3_storage_fs.isfile(posixpath.join(path, "model.mlem")) assert s3_storage_fs.isfile(model_path + MLEM_EXT) assert s3_storage_fs.isfile(model_path) diff --git a/tests/core/test_objects.py b/tests/core/test_objects.py index 290b293d..d6775aa3 100644 --- a/tests/core/test_objects.py +++ b/tests/core/test_objects.py @@ -11,7 +11,7 @@ from mlem.core.artifacts import Artifacts, LocalArtifact, Storage from mlem.core.errors import MlemProjectNotFound, WrongRequirementsError -from mlem.core.meta_io import MLEM_DIR, MLEM_EXT +from mlem.core.meta_io import MLEM_EXT from mlem.core.metadata import load, load_meta from mlem.core.model import ModelIO, ModelType from mlem.core.objects import ( @@ -75,12 +75,9 @@ def get(name): return get -@pytest.mark.parametrize("external", [True, False]) -def test_meta_dump_curdir(meta, mlem_curdir_project, external): - meta.dump(DEPLOY_NAME, external=external) +def test_meta_dump_curdir(meta, mlem_curdir_project): + meta.dump(DEPLOY_NAME) path = DEPLOY_NAME + MLEM_EXT - if not external: - path = os.path.join(MLEM_DIR, meta.object_type, path) assert os.path.isfile(path) assert isinstance(load(DEPLOY_NAME), MlemDeployment) @@ -90,92 +87,32 @@ def test_meta_dump__no_root(meta, tmpdir): meta.dump(DEPLOY_NAME, project=str(tmpdir)) -def test_meta_dump_fullpath_in_project_no_link(mlem_project, meta): - meta.dump( - os.path.join(mlem_project, MLEM_DIR, meta.object_type, DEPLOY_NAME), - index=True, - external=True, - ) - link_path = os.path.join( - mlem_project, MLEM_DIR, MlemLink.object_type, DEPLOY_NAME + MLEM_EXT - ) - assert not os.path.exists(link_path) - - -def test_meta_dump_internal(mlem_project, meta, path_and_root): - path, root = path_and_root(DEPLOY_NAME) - meta.dump(path, project=root, external=False) - assert meta.name == DEPLOY_NAME - meta_path = os.path.join( - mlem_project, - MLEM_DIR, - MlemDeployment.object_type, - DEPLOY_NAME + MLEM_EXT, - ) - assert os.path.isfile(meta_path) - load_path = load_meta(meta_path) - assert isinstance(load_path, MlemDeployment) - assert load_path.name == meta.name - load_root = load_meta(path, project=root) - assert isinstance(load_root, MlemDeployment) - assert load_root.name == meta.name - - def test_meta_dump_external(mlem_project, meta, path_and_root): path, root = path_and_root(DEPLOY_NAME) - meta.dump(path, project=root, external=True) + meta.dump(path, project=root) assert meta.name == DEPLOY_NAME meta_path = os.path.join(mlem_project, DEPLOY_NAME + MLEM_EXT) assert os.path.isfile(meta_path) loaded = load_meta(meta_path) assert isinstance(loaded, MlemDeployment) assert loaded.name == meta.name - link_path = os.path.join( - mlem_project, MLEM_DIR, MlemLink.object_type, DEPLOY_NAME + MLEM_EXT - ) - assert os.path.isfile(link_path) - assert isinstance(load_meta(link_path, follow_links=False), MlemLink) -@pytest.mark.parametrize("external", [False, True]) -def test_model_dump_curdir(model_meta, mlem_curdir_project, external): - model_meta.dump(MODEL_NAME, external=external) +def test_model_dump_curdir(model_meta, mlem_curdir_project): + model_meta.dump(MODEL_NAME) assert model_meta.name == MODEL_NAME - if not external: - prefix = Path(os.path.join(MLEM_DIR, model_meta.object_type)) - else: - prefix = Path("") - assert os.path.isfile(prefix / MODEL_NAME) - assert os.path.isfile(prefix / (MODEL_NAME + MLEM_EXT)) + assert os.path.isfile(MODEL_NAME) + assert os.path.isfile(MODEL_NAME + MLEM_EXT) assert isinstance(load_meta(MODEL_NAME), MlemModel) -def test_model_dump_internal(mlem_project, model_meta, path_and_root): - path, root = path_and_root(MODEL_NAME) - model_meta.dump(path, project=root, external=False) - assert model_meta.name == MODEL_NAME - model_path = os.path.join( - mlem_project, MLEM_DIR, MlemModel.object_type, MODEL_NAME - ) - assert os.path.isfile(model_path + MLEM_EXT) - assert os.path.isfile(model_path) - - def test_model_dump_external(mlem_project, model_meta, path_and_root): path, root = path_and_root(MODEL_NAME) - model_meta.dump(path, project=root, external=True) + model_meta.dump(path, project=root) assert model_meta.name == MODEL_NAME model_path = os.path.join(mlem_project, MODEL_NAME) assert os.path.isfile(model_path + MLEM_EXT) assert os.path.isfile(model_path) - link_path = os.path.join( - mlem_project, MLEM_DIR, MlemLink.object_type, MODEL_NAME + MLEM_EXT - ) - assert os.path.isfile(link_path) - link = load_meta(link_path, follow_links=False) - assert isinstance(link, MlemLink) - model = link.load_link() - assert model.dict() == model_meta.dict() def _check_cloned_model(cloned_model_meta: MlemObject, path, fs=None): @@ -224,7 +161,7 @@ def test_model_cloning(model_single_path): model = load_meta(model_single_path) with tempfile.TemporaryDirectory() as path: path = posixpath.join(path, "cloned") - model.clone(path, index=False) + model.clone(path) cloned_model_meta = load_meta(path, load_value=False) _check_cloned_model(cloned_model_meta, path) @@ -234,24 +171,18 @@ def test_complex_model_cloning(complex_model_single_path): model = load_meta(complex_model_single_path) with tempfile.TemporaryDirectory() as path: path = posixpath.join(path, "cloned") - model.clone(path, index=False) + model.clone(path) cloned_model_meta = load_meta(path, load_value=False) _check_complex_cloned_model(cloned_model_meta, path) -@pytest.mark.parametrize("external", [True, False]) -def test_model_cloning_to_project(model_single_path, mlem_project, external): +def test_model_cloning_to_project(model_single_path, mlem_project): model = load_meta(model_single_path) - model.clone("model", project=mlem_project, index=False, external=external) + model.clone("model", project=mlem_project) cloned_model_meta = load_meta( "model", project=mlem_project, load_value=False ) - if external: - path = os.path.join(mlem_project, "model") - else: - path = os.path.join( - mlem_project, MLEM_DIR, MlemModel.object_type, "model" - ) + path = os.path.join(mlem_project, "model") _check_cloned_model(cloned_model_meta, path) @@ -259,7 +190,7 @@ def test_model_cloning_to_project(model_single_path, mlem_project, external): def test_model_cloning_to_remote(model_path, s3_tmp_path, s3_storage_fs): model = load_meta(model_path) path = s3_tmp_path("model_cloning_to_remote") - model.clone(path, index=False) + model.clone(path) s3path = path[len("s3:/") :] assert s3_storage_fs.isfile(s3path + MLEM_EXT) assert s3_storage_fs.isfile(s3path) @@ -287,7 +218,7 @@ def get(project="simple"): def test_remote_model_cloning(remote_model_meta, project): with tempfile.TemporaryDirectory() as path: path = os.path.join(path, "model") - remote_model_meta(project).clone(path, index=False) + remote_model_meta(project).clone(path) cloned_model_meta = load_meta(path, load_value=False) _check_cloned_model(cloned_model_meta, path) @@ -302,7 +233,7 @@ def test_remote_model_cloning_to_remote( remote_model_meta, project, s3_tmp_path, s3_storage_fs ): path = s3_tmp_path("remote_model_cloning_to_remote") - remote_model_meta(project).clone(path, index=False) + remote_model_meta(project).clone(path) s3path = path[len("s3:/") :] assert s3_storage_fs.isfile(s3path + MLEM_EXT) assert s3_storage_fs.isfile(s3path) @@ -336,9 +267,7 @@ def test_double_link_load(filled_mlem_project): latest = load_meta( "latest", project=filled_mlem_project, follow_links=False ) - link = latest.make_link( - "external", project=filled_mlem_project, external=True - ) + link = latest.make_link("external", project=filled_mlem_project) assert link.link_type == "model" model = load_meta( "external", project=filled_mlem_project, follow_links=True @@ -379,7 +308,7 @@ def test_link_dump_in_mlem(model_path_mlem_project): link_type="model", ) link_name = "latest" - link.dump(link_name, project=mlem_project, external=True, index=False) + link.dump(link_name, project=mlem_project) model = load_meta(os.path.join(mlem_project, link_name), follow_links=True) assert isinstance(model, MlemModel) @@ -401,13 +330,9 @@ def test_mlem_project_root(filled_mlem_project): path = Path(filled_mlem_project) assert os.path.exists(path) assert os.path.isdir(path) - mlem_dir = path / MLEM_DIR - assert os.path.isdir(mlem_dir) - assert os.path.isfile(mlem_dir / "link" / ("model1" + MLEM_EXT)) - assert os.path.isfile(mlem_dir / "link" / ("latest" + MLEM_EXT)) - model_dir = path / "model1" - assert os.path.isfile(str(model_dir) + MLEM_EXT) - assert os.path.isfile(str(model_dir)) + assert os.path.isfile(path / ("model1" + MLEM_EXT)) + assert os.path.isfile(path / ("latest" + MLEM_EXT)) + assert os.path.isfile(path / "model1") class MockModelIO(ModelIO): diff --git a/tests/resources/empty/.mlem/config.yaml b/tests/resources/empty/.mlem.yaml similarity index 100% rename from tests/resources/empty/.mlem/config.yaml rename to tests/resources/empty/.mlem.yaml diff --git a/tests/resources/storage/.mlem/config.yaml b/tests/resources/storage/.mlem.yaml similarity index 100% rename from tests/resources/storage/.mlem/config.yaml rename to tests/resources/storage/.mlem.yaml diff --git a/tests/test_config.py b/tests/test_config.py index 4a6c4e83..1b105f99 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,7 +1,7 @@ import posixpath -from mlem.config import CONFIG_FILE_NAME, MlemConfig, project_config -from mlem.constants import MLEM_DIR +from mlem.config import MlemConfig, project_config +from mlem.constants import MLEM_CONFIG_FILE_NAME from mlem.contrib.fastapi import FastAPIServer from mlem.core.artifacts import FSSpecStorage, LocalStorage from mlem.core.meta_io import get_fs @@ -25,7 +25,7 @@ def test_loading_empty(set_mlem_project_root): def test_loading_remote(s3_tmp_path, s3_storage_fs): project = s3_tmp_path("remote_conf") fs, path = get_fs(project) - path = posixpath.join(path, MLEM_DIR, CONFIG_FILE_NAME) + path = posixpath.join(path, MLEM_CONFIG_FILE_NAME) with fs.open(path, "w") as f: f.write("core:\n ADDITIONAL_EXTENSIONS: ext1\n") assert project_config(path, fs=fs).additional_extensions == ["ext1"]