Skip to content
This repository has been archived by the owner on Sep 13, 2023. It is now read-only.

Get rid of mlem dir #395

Merged
merged 7 commits into from
Oct 12, 2022
Merged

Get rid of mlem dir #395

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
46bb0e4
get rid of mlem dir
mike0sv Sep 2, 2022
3521de3
Le Merge
mike0sv Oct 7, 2022
caba5ea
fix tests
mike0sv Oct 7, 2022
c913195
fix bitbucket
mike0sv Oct 7, 2022
3c28409
fix gitlab
mike0sv Oct 7, 2022
9d311e9
fix other tests
mike0sv Oct 7, 2022
7eb35c9
fix bb tests
mike0sv Oct 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions mlem/api/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,13 @@
import_object,
init,
link,
ls,
serve,
)

__all__ = [
"save",
"load",
"load_meta",
"ls",
"clone",
"init",
"link",
Expand Down
62 changes: 9 additions & 53 deletions mlem/api/commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
MLEM's Python API
"""
import posixpath
from typing import Any, Dict, Iterable, List, Optional, Type, Union
from typing import Any, Dict, Optional, Union

from fsspec import AbstractFileSystem
from fsspec.implementations.local import LocalFileSystem
Expand All @@ -14,8 +14,7 @@
get_model_meta,
parse_import_type_modifier,
)
from mlem.config import CONFIG_FILE_NAME, project_config
from mlem.constants import PREDICT_METHOD_NAME
from mlem.constants import MLEM_CONFIG_FILE_NAME, PREDICT_METHOD_NAME
from mlem.core.errors import (
InvalidArgumentError,
MlemError,
Expand All @@ -25,7 +24,7 @@
WrongMethodError,
)
from mlem.core.import_objects import ImportAnalyzer, ImportHook
from mlem.core.meta_io import MLEM_DIR, Location, get_fs
from mlem.core.meta_io import Location, get_fs
from mlem.core.metadata import load_meta, save
from mlem.core.objects import (
MlemBuilder,
Expand Down Expand Up @@ -56,8 +55,6 @@ def apply(
method: str = None,
output: str = None,
target_project: str = None,
index: bool = None,
external: bool = None,
batch_size: Optional[int] = None,
) -> Optional[Any]:
"""Apply provided model against provided data
Expand All @@ -70,8 +67,6 @@ def apply(
If more than one is available, will fail.
output (str, optional): If value is provided,
assume it's path and save output there.
index (bool): Whether to index saved output in MLEM root folder.
external (bool): Whether to save result outside mlem dir

Returns:
If `output=None`, returns results for given data.
Expand Down Expand Up @@ -103,9 +98,7 @@ def apply(
return res
if len(res) == 1:
res = res[0]
return save(
res, output, project=target_project, external=external, index=index
)
return save(res, output, project=target_project)


def apply_remote(
Expand All @@ -114,7 +107,6 @@ def apply_remote(
method: str = None,
output: str = None,
target_project: str = None,
index: bool = False,
**client_kwargs,
) -> Optional[Any]:
"""Apply provided model against provided data
Expand All @@ -127,7 +119,6 @@ def apply_remote(
If more than one is available, will fail.
output (str, optional): If value is provided,
assume it's path and save output there.
index (bool): Whether to index saved output in MLEM root folder.

Returns:
If `output=None`, returns results for given data.
Expand All @@ -151,7 +142,7 @@ def apply_remote(
return res
if len(res) == 1:
res = res[0]
return save(res, output, project=target_project, index=index)
return save(res, output, project=target_project)


def clone(
Expand All @@ -164,8 +155,6 @@ def clone(
target_fs: Optional[str] = None,
follow_links: bool = True,
load_value: bool = False,
index: bool = None,
external: bool = None,
) -> MlemObject:
"""Clones MLEM object from `path` to `out`
and returns Python representation for the created object
Expand All @@ -181,8 +170,6 @@ def clone(
follow_links (bool, optional): If object we read is a MLEM link, whether to load
the actual object link points to. Defaults to True.
load_value (bool, optional): Load actual python object incorporated in MlemObject. Defaults to False.
index: whether to index object in target project
external: wheter to put object inside mlem dir in target project

Returns:
MlemObject: Copy of initial object saved to `out`
Expand All @@ -202,14 +189,12 @@ def clone(
target,
fs=target_fs,
project=target_project,
index=index,
external=external,
)


def init(path: str = ".") -> None:
"""Creates .mlem directory in `path`"""
path = posixpath.join(path, MLEM_DIR)
"""Creates mlem config in `path`"""
path = posixpath.join(path, MLEM_CONFIG_FILE_NAME)
Copy link
Contributor

@aguschin aguschin Oct 7, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know the file is called .mlem.yaml. But for GTO it's .gto. Should we rename it to .gto.yaml in GTO, or should we rename .mlem.yaml to .mlem? We shouldn't confuse users with different approaches. Did we had same files in other tools maybe? cc @shcheklein

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also if users have .mlem dir now, it's not possible to create .mlem file. So renaming config file will force them to move everything outside of .mlem dir and change their workflows maybe.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mike0sv if everything else is good, let's merge this so we won't get a PR queue again? And then discuss it separately.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

my 2 cents - if the file is indeed a yaml, gto.yaml > [something.]gto
extensions are for formats :)
as long as gto doesn't invent a new file format/structure but respects yaml, that's the least surprising thing to do

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, then it's decided I guess. Let's keep it here, and I'll update it in GTO as well some time later.

fs, path = get_fs(path)
if fs.exists(path):
echo(
Expand Down Expand Up @@ -252,9 +237,8 @@ def init(path: str = ".") -> None:
"<https://mlem.ai/docs/user-guide/analytics>"
)
)
fs.makedirs(path)
# some fs dont support creating empty dirs
with fs.open(posixpath.join(path, CONFIG_FILE_NAME), "w"):
with fs.open(path, "w"):
pass
echo(
EMOJI_MLEM
Expand All @@ -273,7 +257,6 @@ def link(
rev: Optional[str] = None,
target: Optional[str] = None,
target_project: Optional[str] = None,
external: Optional[bool] = None,
follow_links: bool = True,
absolute: bool = False,
) -> MlemLink:
Expand All @@ -288,7 +271,6 @@ def link(
treat `target` as link name and dump link in MLEM DIR
follow_links (bool): Whether to make link to the underlying object
if `source` is itself a link. Defaults to True.
external (bool): Whether to save link outside mlem dir
absolute (bool): Whether to make link absolute or relative to mlem project

Returns:
Expand All @@ -308,7 +290,6 @@ def link(
return source.make_link(
target,
project=target_project,
external=external,
absolute=absolute,
)

Expand Down Expand Up @@ -359,25 +340,6 @@ def _validate_ls_project(loc: Location, project):
mlem_project_exists(loc.project, loc.fs, raise_on_missing=True)


def ls( # pylint: disable=too-many-locals
project: str = ".",
rev: Optional[str] = None,
fs: Optional[AbstractFileSystem] = None,
type_filter: Union[
Type[MlemObject], Iterable[Type[MlemObject]], None
] = None,
include_links: bool = True,
ignore_errors: bool = False,
) -> Dict[Type[MlemObject], List[MlemObject]]:
loc = Location.resolve(
"", project=project, rev=rev, fs=fs, find_project=True
)
_validate_ls_project(loc, project)
return project_config(project, fs).index.list(
loc, type_filter, include_links, ignore_errors
)


def import_object(
path: str,
project: Optional[str] = None,
Expand All @@ -388,8 +350,6 @@ def import_object(
target_fs: Optional[AbstractFileSystem] = None,
type_: Optional[str] = None,
copy_data: bool = True,
external: bool = None,
index: bool = None,
):
"""Try to load an object as MLEM model (or data) and return it,
optionally saving to the specified target location
Expand All @@ -408,8 +368,6 @@ def import_object(
target,
fs=target_fs,
project=target_project,
index=index,
external=external,
)
return meta

Expand All @@ -421,8 +379,6 @@ def deploy(
project: Optional[str] = None,
rev: Optional[str] = None,
fs: Optional[AbstractFileSystem] = None,
external: bool = None,
index: bool = None,
env_kwargs: Dict[str, Any] = None,
**deploy_kwargs,
) -> MlemDeployment:
Expand Down Expand Up @@ -456,7 +412,7 @@ def deploy(
env=env,
**deploy_kwargs,
)
deploy_meta.dump(deploy_meta_or_path, fs, project, index, external)
deploy_meta.dump(deploy_meta_or_path, fs, project)
else:
deploy_meta = deploy_meta_or_path
update = True
Expand Down
3 changes: 1 addition & 2 deletions mlem/cli/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from mlem.cli.deployment import deployment
from mlem.cli.dev import dev
from mlem.cli.import_object import import_object
from mlem.cli.info import ls, pretty_print
from mlem.cli.info import pretty_print
from mlem.cli.init import init
from mlem.cli.link import link
from mlem.cli.main import app
Expand All @@ -25,7 +25,6 @@
"build",
"pretty_print",
"link",
"ls",
"clone",
"serve",
"config",
Expand Down
14 changes: 0 additions & 14 deletions mlem/cli/apply.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@
option_data,
option_data_project,
option_data_rev,
option_external,
option_file_conf,
option_index,
option_json,
option_load,
option_method,
Expand Down Expand Up @@ -80,8 +78,6 @@ def apply(
import_: bool = option_import,
import_type: str = option_import_type,
batch_size: Optional[int] = option_batch_size,
index: bool = option_index,
external: bool = option_external,
json: bool = option_json,
):
"""Apply a model to data. The result will be saved as a MLEM object to `output` if
Expand Down Expand Up @@ -116,8 +112,6 @@ def apply(
data,
method=method,
output=output,
index=index,
external=external,
batch_size=batch_size,
)
if output is None and json:
Expand All @@ -144,7 +138,6 @@ def _apply_remote(
data,
project,
rev,
index,
method,
output,
target_project,
Expand All @@ -169,7 +162,6 @@ def _apply_remote(
data,
project,
rev,
index,
method,
output,
target_project,
Expand All @@ -190,15 +182,13 @@ def apply_remote_load(
output: Optional[str] = option_output,
target_project: Optional[str] = option_target_project,
method: str = option_method,
index: bool = option_index,
json: bool = option_json,
load: Optional[str] = option_load("client"),
):
return _apply_remote(
data,
project,
rev,
index,
method,
output,
target_project,
Expand Down Expand Up @@ -229,7 +219,6 @@ def apply_remote_func(
output: Optional[str] = option_output,
target_project: Optional[str] = option_target_project,
method: str = option_method,
index: bool = option_index,
json: bool = option_json,
file_conf: List[str] = option_file_conf("client"),
**__kwargs__,
Expand All @@ -238,7 +227,6 @@ def apply_remote_func(
data,
project,
rev,
index,
method,
output,
target_project,
Expand All @@ -255,7 +243,6 @@ def run_apply_remote(
data_path: str,
project,
rev,
index,
method,
output,
target_project,
Expand All @@ -275,6 +262,5 @@ def run_apply_remote(
method=method,
output=output,
target_project=target_project,
index=index,
)
return result
6 changes: 0 additions & 6 deletions mlem/cli/clone.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@

from mlem.cli.main import (
mlem_command,
option_external,
option_index,
option_project,
option_rev,
option_target_project,
Expand All @@ -19,8 +17,6 @@ def clone(
project: Optional[str] = option_project,
rev: Optional[str] = option_rev,
target_project: Optional[str] = option_target_project,
external: Optional[bool] = option_external,
index: Optional[bool] = option_index,
):
"""Copy a MLEM Object from `uri` and
saves a copy of it to `target` path.
Expand All @@ -33,6 +29,4 @@ def clone(
project=project,
rev=rev,
target_project=target_project,
external=external,
index=index,
)
Loading