Skip to content

Commit

Permalink
AWS focussed CLI implementation for Shareable Viz (#1661)
Browse files Browse the repository at this point in the history
Add `kedro viz deploy` as a cli command to share Kedro-viz via AWS
  • Loading branch information
ravi-kumar-pilla authored Dec 15, 2023
1 parent f483b30 commit 6547213
Show file tree
Hide file tree
Showing 11 changed files with 291 additions and 33 deletions.
2 changes: 0 additions & 2 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,6 @@ Finally, you can use pseudo-random data, which is procedurally-generated on page

### Launch a development server with a real Kedro project

> **Note**: Kedro-Viz>=3.8.0 will not work with projects created with Kedro<=0.16.6. Please consider migrating your project to Kedro>=0.17.0 before you develop against the latest version of Kedro-Viz.
> **Note**: Kedro-Viz>=7.0.0 will not work with projects created with Kedro<=0.17.0. Please consider migrating your project to Kedro>=0.18.0 before you develop against the latest version of Kedro-Viz.

Expand Down
18 changes: 18 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,24 @@ Options:
-h, --help Show this message and exit.
```
To deploy Kedro-Viz from the command line as a Kedro plugin, use the following command from the root folder of your Kedro project:
```bash
kedro viz deploy
```
```bash
Usage: kedro viz deploy [OPTIONS]

Deploy and host Kedro Viz on AWS S3.

Options:
--region TEXT AWS region where your S3 bucket is located [required]
--bucket-name TEXT AWS S3 bucket name where Kedro Viz will be hosted
[required]
-h, --help Show this message and exit.
```
### Experiment Tracking usage
To enable [experiment tracking](https://docs.kedro.org/en/stable/experiment_tracking/index.html) in Kedro-Viz, you need to add the Kedro-Viz `SQLiteStore` to your Kedro project.
Expand Down
1 change: 1 addition & 0 deletions RELEASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ Please follow the established format:
## Major features and improvements

- Display hosted URL in CLI while launching kedro viz. (#1644)
- AWS focussed CLI implementation `kedro viz deploy` for shareable viz. (#1661)

## Bug fixes and other changes

Expand Down
6 changes: 6 additions & 0 deletions docs/source/share_kedro_viz.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@ Here's an example of the flow:

![](./images/kedro-publish-share.gif)

From Kedro-Viz version 7.0.0, you can now publish and share your Kedro-Viz project from the command line. Use the following command from the root folder of your Kedro project

```bash
kedro viz deploy --region=[aws-bucket-region] --bucket-name=[aws-bucket-name]
```

## Permissions and access control

All permissions and access control are controlled by AWS. It's up to you, the user, if you want to allow anyone to see your project or limit access to certain IP addresses, users, or groups.
Expand Down
35 changes: 35 additions & 0 deletions package/kedro_viz/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,38 @@

DEFAULT_HOST = "127.0.0.1"
DEFAULT_PORT = 4141

VIZ_DEPLOY_TIME_LIMIT = 60

AWS_REGIONS = [
"us-east-2",
"us-east-1",
"us-west-1",
"us-west-2",
"af-south-1",
"ap-east-1",
"ap-south-2",
"ap-southeast-3",
"ap-southeast-4",
"ap-south-1",
"ap-northeast-3",
"ap-northeast-2",
"ap-southeast-1",
"ap-southeast-2",
"ap-northeast-1",
"ca-central-1",
"cn-north-1",
"cn-northwest-1",
"eu-central-1",
"eu-west-1",
"eu-west-2",
"eu-south-1",
"eu-west-3",
"eu-north-1",
"eu-south-2",
"eu-central-2",
"sa-east-1",
"me-south-1",
"me-central-1",
"il-central-1",
]
14 changes: 3 additions & 11 deletions package/kedro_viz/integrations/kedro/data_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,6 @@ def __call__(self, *args, **kwargs):
pass


def _bootstrap(project_path: Path):
"""Bootstrap the integration by running various Kedro bootstrapping methods
depending on the version
"""
from kedro.framework.startup import bootstrap_project

bootstrap_project(project_path)


def _get_dataset_stats(project_path: Path) -> Dict:
"""Return the stats saved at stats.json as a dictionary if found.
If not, return an empty dictionary
Expand Down Expand Up @@ -112,9 +103,10 @@ def load_data(
A tuple containing the data catalog and the pipeline dictionary
and the session store.
"""
_bootstrap(project_path)

from kedro.framework.project import pipelines
from kedro.framework.startup import bootstrap_project

bootstrap_project(project_path)

with KedroSession.create(
project_path=project_path,
Expand Down
81 changes: 79 additions & 2 deletions package/kedro_viz/launchers/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,16 @@
from watchgod import RegExpWatcher, run_process

from kedro_viz import __version__
from kedro_viz.constants import DEFAULT_HOST, DEFAULT_PORT
from kedro_viz.constants import AWS_REGIONS, DEFAULT_HOST, DEFAULT_PORT
from kedro_viz.integrations.deployment.s3_deployer import S3Deployer
from kedro_viz.integrations.pypi import get_latest_version, is_running_outdated_version
from kedro_viz.launchers.utils import _check_viz_up, _start_browser, _wait_for
from kedro_viz.launchers.utils import (
_check_viz_up,
_start_browser,
_wait_for,
viz_deploy_progress_timer,
)
from kedro_viz.server import load_and_populate_data

_VIZ_PROCESSES: Dict[str, int] = {}

Expand Down Expand Up @@ -193,3 +200,73 @@ def run(
except Exception as ex: # pragma: no cover
traceback.print_exc()
raise KedroCliError(str(ex)) from ex


@viz.command(context_settings={"help_option_names": ["-h", "--help"]})
@click.option(
"--region",
type=str,
required=True,
help="AWS region where your S3 bucket is located",
)
@click.option(
"--bucket-name",
type=str,
required=True,
help="AWS S3 bucket name where Kedro Viz will be hosted",
)
def deploy(region, bucket_name):
"""Deploy and host Kedro Viz on AWS S3"""
if region not in AWS_REGIONS:
click.echo(
click.style(
"ERROR: Invalid AWS region. Please enter a valid AWS Region (eg., us-east-2).\n"
"Please find the complete list of available regions at :\n"
"https://docs.aws.amazon.com/AmazonRDS/latest"
"/UserGuide/Concepts.RegionsAndAvailabilityZones.html"
"#Concepts.RegionsAndAvailabilityZones.Regions",
fg="red",
),
)
return

try:
viz_deploy_timer = multiprocessing.Process(target=viz_deploy_progress_timer)
viz_deploy_timer.start()

# Loads and populates data from underlying Kedro Project
load_and_populate_data(Path.cwd(), ignore_plugins=True)

# Start the deployment
deployer = S3Deployer(region, bucket_name)
url = deployer.deploy_and_get_url()

click.echo(
click.style(
"\u2728 Success! Kedro Viz has been deployed on AWS S3. It can be accessed at :\n"
f"{url}",
fg="green",
),
)
except PermissionError: # pragma: no cover
click.echo(
click.style(
"PERMISSION ERROR: Deploying and hosting Kedro-Viz requires "
"AWS access keys, a valid AWS region and bucket name.\n"
"Please supply your AWS access keys as environment variables "
"and make sure the AWS region and bucket name are valid.\n"
"More information can be found at : "
"https://docs.kedro.org/en/stable/visualisation/share_kedro_viz.html",
fg="red",
)
)
# pylint: disable=broad-exception-caught
except Exception as exc: # pragma: no cover
click.echo(
click.style(
f"ERROR: Failed to deploy and host Kedro-Viz on AWS S3 : {exc} ",
fg="red",
)
)
finally:
viz_deploy_timer.terminate()
14 changes: 14 additions & 0 deletions package/kedro_viz/launchers/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

import requests

from kedro_viz.constants import VIZ_DEPLOY_TIME_LIMIT

logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -90,3 +92,15 @@ def _start_browser(host: str, port: int):

if _is_localhost(host):
webbrowser.open_new(f"http://{host}:{port}/")


def viz_deploy_progress_timer():
"""Shows progress timer and message for kedro viz deploy"""
seconds = 0
try:
while seconds <= VIZ_DEPLOY_TIME_LIMIT:
print(f"...Creating your webpage ({seconds}s)", end="\r", flush=True)
sleep(1)
seconds += 1
except KeyboardInterrupt: # pragma: no cover
print("\nCreating your webpage interrupted. Exiting...")
40 changes: 26 additions & 14 deletions package/kedro_viz/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,30 @@ def populate_data(
data_access_manager.add_pipelines(pipelines)


def load_and_populate_data(
path: Path,
env: Optional[str] = None,
ignore_plugins: bool = False,
extra_params: Optional[Dict[str, Any]] = None,
pipeline_name: Optional[str] = None,
):
"""Loads underlying Kedro project data and populates Kedro Viz Repositories"""

# Loads data from underlying Kedro Project
catalog, pipelines, session_store, stats_dict = kedro_data_loader.load_data(
path, env, ignore_plugins, extra_params
)

pipelines = (
pipelines
if pipeline_name is None
else {pipeline_name: pipelines[pipeline_name]}
)

# Creates data repositories which are used by Kedro Viz Backend APIs
populate_data(data_access_manager, catalog, pipelines, session_store, stats_dict)


def run_server(
host: str = DEFAULT_HOST,
port: int = DEFAULT_PORT,
Expand All @@ -58,7 +82,7 @@ def run_server(
autoreload: bool = False,
ignore_plugins: bool = False,
extra_params: Optional[Dict[str, Any]] = None,
): # pylint: disable=redefined-outer-name, too-many-locals
): # pylint: disable=redefined-outer-name
"""Run a uvicorn server with a FastAPI app that either launches API response data from a file
or from reading data from a real Kedro project.
Expand All @@ -83,19 +107,7 @@ def run_server(
path = Path(project_path) if project_path else Path.cwd()

if load_file is None:
# Loads data from underlying Kedro Project
catalog, pipelines, session_store, stats_dict = kedro_data_loader.load_data(
path, env, ignore_plugins, extra_params
)
pipelines = (
pipelines
if pipeline_name is None
else {pipeline_name: pipelines[pipeline_name]}
)
# Creates data repositories which are used by Kedro Viz Backend APIs
populate_data(
data_access_manager, catalog, pipelines, session_store, stats_dict
)
load_and_populate_data(path, env, ignore_plugins, extra_params, pipeline_name)

if save_file:
save_api_responses_to_fs(save_file)
Expand Down
Loading

0 comments on commit 6547213

Please sign in to comment.