Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix mounts for --volume=HOME #977

Merged
merged 2 commits into from
Aug 26, 2019
Merged

Fix mounts for --volume=HOME #977

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
672d954
fix mounts --volume=HOME
Aug 22, 2019
5d0ac60
add changelog
Aug 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,8 @@ neuro job run [OPTIONS] IMAGE [CMD]...

# Starts a container pytorch:latest on a machine with smaller GPU resources
# (see exact values in `neuro config show`) and with two volumes mounted:
# storage://~ --> /var/storage/home (in read-write mode),
# storage://neuromation --> /var/storage/neuromation (in read-only mode).
# storage://<home-directory> --> /var/storage/home (in read-write mode),
# storage://neuromation/public --> /var/storage/neuromation (in read-only mode).
neuro run --preset=gpu-small --volume=HOME pytorch:latest

# Starts a container using the custom image my-ubuntu:latest stored in neuromation
Expand All @@ -215,7 +215,7 @@ Name | Description|
|_\-n, --name NAME_|Optional job name|
|_\-d, --description DESC_|Optional job description in free format|
|_\-q, --quiet_|Run command in quiet mode \(DEPRECATED)|
|_\-v, --volume MOUNT_|Mounts directory from vault into container. Use multiple options to mount more than one volume. --volume=HOME is an alias for storage://~:/var/storage/home:rw and storage://neuromation:/var/storage/neuromation:ro|
|_\-v, --volume MOUNT_|Mounts directory from vault into container. Use multiple options to mount more than one volume. --volume=HOME is an alias for storage://~:/var/storage/home:rw and storage://neuromation/public:/var/storage/neuromation:ro|
|_--entrypoint TEXT_|Executable entrypoint in the container \(note that it overwrites `ENTRYPOINT` and `CMD` instructions of the docker image)|
|_\-e, --env VAR=VAL_|Set environment variable in container Use multiple options to define more than one variable|
|_\--env-file PATH_|File with environment variables to pass|
Expand Down Expand Up @@ -268,7 +268,7 @@ Name | Description|
|_\-n, --name NAME_|Optional job name|
|_\-d, --description DESC_|Optional job description in free format|
|_\-q, --quiet_|Run command in quiet mode \(DEPRECATED)|
|_\-v, --volume MOUNT_|Mounts directory from vault into container. Use multiple options to mount more than one volume. --volume=HOME is an alias for storage://~:/var/storage/home:rw and storage://neuromation:/var/storage/neuromation:ro|
|_\-v, --volume MOUNT_|Mounts directory from vault into container. Use multiple options to mount more than one volume. --volume=HOME is an alias for storage://~:/var/storage/home:rw and storage://neuromation/public:/var/storage/neuromation:ro|
|_--entrypoint TEXT_|Executable entrypoint in the container \(note that it overwrites `ENTRYPOINT` and `CMD` instructions of the docker image)|
|_\-e, --env VAR=VAL_|Set environment variable in container Use multiple options to define more than one variable|
|_\--env-file PATH_|File with environment variables to pass|
Expand Down Expand Up @@ -1284,8 +1284,8 @@ neuro run [OPTIONS] IMAGE [CMD]...

# Starts a container pytorch:latest on a machine with smaller GPU resources
# (see exact values in `neuro config show`) and with two volumes mounted:
# storage://~ --> /var/storage/home (in read-write mode),
# storage://neuromation --> /var/storage/neuromation (in read-only mode).
# storage://<home-directory> --> /var/storage/home (in read-write mode),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

side change aiming improvement of the readability

# storage://neuromation/public --> /var/storage/neuromation (in read-only mode).
neuro run --preset=gpu-small --volume=HOME pytorch:latest

# Starts a container using the custom image my-ubuntu:latest stored in neuromation
Expand All @@ -1305,7 +1305,7 @@ Name | Description|
|_\-n, --name NAME_|Optional job name|
|_\-d, --description DESC_|Optional job description in free format|
|_\-q, --quiet_|Run command in quiet mode \(DEPRECATED)|
|_\-v, --volume MOUNT_|Mounts directory from vault into container. Use multiple options to mount more than one volume. --volume=HOME is an alias for storage://~:/var/storage/home:rw and storage://neuromation:/var/storage/neuromation:ro|
|_\-v, --volume MOUNT_|Mounts directory from vault into container. Use multiple options to mount more than one volume. --volume=HOME is an alias for storage://~:/var/storage/home:rw and storage://neuromation/public:/var/storage/neuromation:ro|
|_--entrypoint TEXT_|Executable entrypoint in the container \(note that it overwrites `ENTRYPOINT` and `CMD` instructions of the docker image)|
|_\-e, --env VAR=VAL_|Set environment variable in container Use multiple options to define more than one variable|
|_\--env-file PATH_|File with environment variables to pass|
Expand Down Expand Up @@ -1358,7 +1358,7 @@ Name | Description|
|_\-n, --name NAME_|Optional job name|
|_\-d, --description DESC_|Optional job description in free format|
|_\-q, --quiet_|Run command in quiet mode \(DEPRECATED)|
|_\-v, --volume MOUNT_|Mounts directory from vault into container. Use multiple options to mount more than one volume. --volume=HOME is an alias for storage://~:/var/storage/home:rw and storage://neuromation:/var/storage/neuromation:ro|
|_\-v, --volume MOUNT_|Mounts directory from vault into container. Use multiple options to mount more than one volume. --volume=HOME is an alias for storage://~:/var/storage/home:rw and storage://neuromation/public:/var/storage/neuromation:ro|
|_--entrypoint TEXT_|Executable entrypoint in the container \(note that it overwrites `ENTRYPOINT` and `CMD` instructions of the docker image)|
|_\-e, --env VAR=VAL_|Set environment variable in container Use multiple options to define more than one variable|
|_\--env-file PATH_|File with environment variables to pass|
Expand Down
10 changes: 5 additions & 5 deletions neuromation/cli/job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ def job() -> None:
help="Mounts directory from vault into container. "
"Use multiple options to mount more than one volume. "
"--volume=HOME is an alias for storage://~:/var/storage/home:rw and "
"storage://neuromation:/var/storage/neuromation:ro",
"storage://neuromation/public:/var/storage/neuromation:ro",
)
@click.option(
"--entrypoint",
Expand Down Expand Up @@ -621,7 +621,7 @@ def format_fail(job: str, reason: Exception) -> str:
help="Mounts directory from vault into container. "
"Use multiple options to mount more than one volume. "
"--volume=HOME is an alias for storage://~:/var/storage/home:rw and "
"storage://neuromation:/var/storage/neuromation:ro",
"storage://neuromation/public:/var/storage/neuromation:ro",
)
@click.option(
"--entrypoint",
Expand Down Expand Up @@ -695,8 +695,8 @@ async def run(

# Starts a container pytorch:latest on a machine with smaller GPU resources
# (see exact values in `neuro config show`) and with two volumes mounted:
# storage://~ --> /var/storage/home (in read-write mode),
# storage://neuromation --> /var/storage/neuromation (in read-only mode).
# storage://<home-directory> --> /var/storage/home (in read-write mode),
# storage://neuromation/public --> /var/storage/neuromation (in read-only mode).
neuro run --preset=gpu-small --volume=HOME pytorch:latest

# Starts a container using the custom image my-ubuntu:latest stored in neuromation
Expand Down Expand Up @@ -809,7 +809,7 @@ async def run_job(
volumes.add(root.client.parse.volume("storage://~:/var/storage/home:rw"))
volumes.add(
root.client.parse.volume(
"storage://neuromation:/var/storage/neuromation:ro"
"storage://neuromation/public:/var/storage/neuromation:ro"
)
)
else:
Expand Down
18 changes: 18 additions & 0 deletions tests/e2e/test_e2e_jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1071,3 +1071,21 @@ def test_job_submit_browse(helper: Helper, fakebrowser: Any) -> None:
)
assert "Browsing https://job-" in captured.out
assert "Open job URL: https://job-" in captured.err


@pytest.mark.e2e
def test_job_run_home_volumes_automount(helper: Helper, fakebrowser: Any) -> None:
command = "[ -d /var/storage/home -a -d /var/storage/neuromation ]"

# first, run without --volume=HOME
helper.run_job_and_wait_state(
image=UBUNTU_IMAGE_NAME, command=command, wait_state=JobStatus.FAILED
)

# then, run with --volume=HOME
helper.run_job_and_wait_state(
image=UBUNTU_IMAGE_NAME,
command=command,
params=("--volume", "HOME"),
wait_state=JobStatus.SUCCEEDED,
)