From 149d0c682431586ebac3b9ffca29ac243ba718d4 Mon Sep 17 00:00:00 2001 From: Romil Date: Sat, 8 Oct 2022 12:26:55 -0700 Subject: [PATCH 1/4] Container example --- examples/docker/echo_app.py | 52 +++++++++++++++++++++++++++++ examples/docker/echo_app.yaml | 27 +++++++++++++++ examples/docker/echo_app/Dockerfile | 7 ++++ examples/docker/echo_app/README.md | 3 ++ examples/docker/echo_app/echo.py | 26 +++++++++++++++ 5 files changed, 115 insertions(+) create mode 100644 examples/docker/echo_app.py create mode 100644 examples/docker/echo_app.yaml create mode 100644 examples/docker/echo_app/Dockerfile create mode 100644 examples/docker/echo_app/README.md create mode 100644 examples/docker/echo_app/echo.py diff --git a/examples/docker/echo_app.py b/examples/docker/echo_app.py new file mode 100644 index 00000000000..0acc105580e --- /dev/null +++ b/examples/docker/echo_app.py @@ -0,0 +1,52 @@ +# Example using SkyPilot python API to run the echo app in a container. +# +# The echo example ingests a file, prints the contents and writes it back out. +# In this YAML, the output is mapped to a Sky Storage object, which writes to a +# cloud bucket. +# +# Usage: +# python echo_app.py + +import random +import sky +import string + +with sky.Dag() as dag: + # The setup command to build the container image + setup = 'docker build -t echo:v0 /echo_app' + + # The command to run - runs the container and mounts volumes + run = 'docker run --rm ' \ + '--volume="/inputs:/inputs:ro" ' \ + '--volume="/outputs:/outputs:rw" ' \ + 'echo:v0 /inputs/README.md /outputs/output.txt' + + echo_app = sky.Task( + setup=setup, + run=run, + ) + + # Configure file mounts to copy local contents to remote + echo_app.set_file_mounts({ + '/inputs': './echo_app', + '/echo_app': './echo_app', + }) + + # Configure outputs for the task - we'll write to a bucket using Sky Storage + output_bucket_name = ''.join(random.choices(string.ascii_lowercase, k=15)) + output_storage = sky.Storage(name=output_bucket_name) + echo_app.set_storage_mounts({ + '/outputs': output_storage, + }) + + # Set resources if required + # echo_app.set_resources({ + # sky.Resources(accelerators='V100'), + # }) + +sky.launch(dag) + +print('Remember to clean up resources after this script is done!\n' + 'Run sky status and sky storage ls to list current resources.\n' + 'Run sky down and sky storage delete to ' + 'delete resources.') diff --git a/examples/docker/echo_app.yaml b/examples/docker/echo_app.yaml new file mode 100644 index 00000000000..7c51881a321 --- /dev/null +++ b/examples/docker/echo_app.yaml @@ -0,0 +1,27 @@ +# Runs the echo example app in a container with custom inputs and outputs +# +# The echo example ingests a file, prints the contents and writes it back out. +# In this YAML, the output is mapped to a Sky Storage object, which writes to a +# cloud bucket. +# +# Usage: +# sky launch -c myclus echo_app.yaml +# sky exec myclus echo_app.yaml +# sky down myclus + +file_mounts: + /inputs: ./echo_app + /echo_app: ./echo_app + /outputs: + name: # Set unique bucket name here! + mode: MOUNT + +setup: | + # Build docker image. If pushed to a registry, can also do docker pull here + docker build -t echo:v0 /echo_app + +run: | + docker run --rm \ + --volume="/inputs:/inputs:ro" \ + --volume="/outputs:/outputs:rw" \ + echo:v0 /inputs/README.md /outputs/output.txt diff --git a/examples/docker/echo_app/Dockerfile b/examples/docker/echo_app/Dockerfile new file mode 100644 index 00000000000..c1f0ba14ef4 --- /dev/null +++ b/examples/docker/echo_app/Dockerfile @@ -0,0 +1,7 @@ +FROM python + +ADD echo.py /app/echo.py + +WORKDIR /app + +ENTRYPOINT ["python", "echo.py"] diff --git a/examples/docker/echo_app/README.md b/examples/docker/echo_app/README.md new file mode 100644 index 00000000000..6031f05a3c8 --- /dev/null +++ b/examples/docker/echo_app/README.md @@ -0,0 +1,3 @@ +# Echo App + +A simple app that ingests a file and writes it out back to a specified path. diff --git a/examples/docker/echo_app/echo.py b/examples/docker/echo_app/echo.py new file mode 100644 index 00000000000..aa1500971c9 --- /dev/null +++ b/examples/docker/echo_app/echo.py @@ -0,0 +1,26 @@ +"""Echo app + +Reads a file, echoes it and writes back to a specified path. +""" +import argparse + + +def main(): + """Main function""" + parser = argparse.ArgumentParser(description='Echo app') + parser.add_argument('input', type=str) + parser.add_argument('output', type=str) + args = parser.parse_args() + + with open(args.input, 'r') as input_file: + content = input_file.read() + print("===== echo app =====") + print("Input file content:") + print(content) + with open(args.output, 'w') as output_file: + output_file.write(content) + print("Output written to {}".format(args.output)) + + +if __name__ == '__main__': + main() From 99a8344d3bb4343612f48e4177f622e4fa255c52 Mon Sep 17 00:00:00 2001 From: Romil Date: Tue, 11 Oct 2022 10:28:45 -0700 Subject: [PATCH 2/4] parenthesis --- examples/docker/echo_app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/docker/echo_app.py b/examples/docker/echo_app.py index 0acc105580e..645f14ce4db 100644 --- a/examples/docker/echo_app.py +++ b/examples/docker/echo_app.py @@ -16,10 +16,10 @@ setup = 'docker build -t echo:v0 /echo_app' # The command to run - runs the container and mounts volumes - run = 'docker run --rm ' \ - '--volume="/inputs:/inputs:ro" ' \ - '--volume="/outputs:/outputs:rw" ' \ - 'echo:v0 /inputs/README.md /outputs/output.txt' + run = ('docker run --rm ', + '--volume="/inputs:/inputs:ro" ', + '--volume="/outputs:/outputs:rw" ', + 'echo:v0 /inputs/README.md /outputs/output.txt') echo_app = sky.Task( setup=setup, From e222e103d1121236f4f02c2ff736dbfcb7ec42a8 Mon Sep 17 00:00:00 2001 From: Romil Date: Tue, 11 Oct 2022 10:40:04 -0700 Subject: [PATCH 3/4] Add explicit StorageMode --- examples/docker/echo_app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/docker/echo_app.py b/examples/docker/echo_app.py index 645f14ce4db..2708270fe4d 100644 --- a/examples/docker/echo_app.py +++ b/examples/docker/echo_app.py @@ -34,7 +34,8 @@ # Configure outputs for the task - we'll write to a bucket using Sky Storage output_bucket_name = ''.join(random.choices(string.ascii_lowercase, k=15)) - output_storage = sky.Storage(name=output_bucket_name) + output_storage = sky.Storage(name=output_bucket_name, + mode=sky.StorageMode.MOUNT) echo_app.set_storage_mounts({ '/outputs': output_storage, }) From e8086b21d382e5ed6207ea3c53a3bc74b1fb0ab2 Mon Sep 17 00:00:00 2001 From: Romil Date: Tue, 11 Oct 2022 11:01:02 -0700 Subject: [PATCH 4/4] lint --- examples/docker/echo_app.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/docker/echo_app.py b/examples/docker/echo_app.py index 2708270fe4d..0edfc5f3928 100644 --- a/examples/docker/echo_app.py +++ b/examples/docker/echo_app.py @@ -16,9 +16,8 @@ setup = 'docker build -t echo:v0 /echo_app' # The command to run - runs the container and mounts volumes - run = ('docker run --rm ', - '--volume="/inputs:/inputs:ro" ', - '--volume="/outputs:/outputs:rw" ', + run = ('docker run --rm --volume="/inputs:/inputs:ro" ' + '--volume="/outputs:/outputs:rw" ' 'echo:v0 /inputs/README.md /outputs/output.txt') echo_app = sky.Task(