diff --git a/.eslintrc.json b/.eslintrc.json index 8c3135079c..ffe6df2b65 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -4,8 +4,7 @@ "curly": ["error"], "valid-typeof": ["error"], "camelcase": "error", - "id-length": ["error", { "min": 3, "exceptions": ["_","a","b","d","e","i","j","k","x","y","id","el","pi","PI","up"] }], - "no-var": ["error"], +"id-length": ["error", { "min": 3, "exceptions": ["_","a","b","d","e","i","j","k","x","y","id","el","pi","PI","up","to"] }], "no-var": ["error"], "lines-between-class-members": ["error", "always"] } } diff --git a/README.npm.md b/README.npm.md index c271389115..038bf75e46 100644 --- a/README.npm.md +++ b/README.npm.md @@ -149,6 +149,7 @@ The example below demonstrates how to configure your kedro-viz using different ` | Name | Type | Default | Description | | ------------ | ------- | ------- | ----------- | | `data` | `{ edges: array (required), layers: array, nodes: array (required), tags: array }` | - | Pipeline data to be displayed on the chart | +| `onActionCallback` | function | - | Callback function to be invoked when the specified action is dispatched. e.g. `const action = { type: NODE_CLICK, payload: node }; onActionCallback(action);` | | options.display | | | | | `expandPipelinesBtn` | boolean | true | Show/Hide expand pipelines button | | `exportBtn` | boolean | true | Show/Hide export button | @@ -166,7 +167,8 @@ The example below demonstrates how to configure your kedro-viz using different ` ### Note -When `display.sidebar` is `false`, `display.miniMap` prop will be ignored. +- `onActionCallback` callback is only called when the user clicks on a node in the flowchart, and we are passing the node object as the payload in the callback argument. In future releases, we will add more actions to be dispatched in this callback. +- When `display.sidebar` is `false`, `display.miniMap` prop will be ignored. All components are annotated to understand their positions in the Kedro-Viz UI. diff --git a/RELEASE.md b/RELEASE.md index 724ac54768..5e08f318d6 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -5,6 +5,33 @@ Please follow the established format: - Use present tense (e.g. 'Add new feature') - Include the ID number for the related PR (or PRs) in parentheses --> +# Release 10.0.0 + +## Major features and improvements + +- Add `kedro viz --lite`, allowing users to run Kedro-Viz without installing Kedro project dependencies. (#1966, #2077, #2093) +- Enable visual slicing of a Kedro pipeline in Kedro-viz. (#2036) +- Improve Kedro-Viz CLI startup time with lazy subcommands and deferring imports (#1920) +- Add documentation for Kedro-viz in VSCode Extension. (#2078) + + +## Bug fixes and other changes + +- Introduce `onActionCallback` prop in Kedro-Viz React component. (#2022) +- Enhance documentation for Dataset previews on Kedro-viz. (#2074) +- Add documentation for Kedro-viz CLI Commands. (#2044) +- Fix design issues in metadata panel. (#2009) +- Fix bug where reloading the page reset to the default pipeline instead of retaining the selected one. (#2041) +- Add feedback component for slicing pipeline. (#2085) +- Add `kedro viz --lite` user warning banner component. (#2092) +- Add `UnavailableDataset` as a default dataset for `--lite` mode. (#2083) +- Fix missing run command in metadata panel for task nodes. (#2055) +- Fix highlight width inconsistency in the Nodelist component.(#2004) +- Migrate `demo.kedro.org` from AWS Lightsail to Github Pages. (#2034, #2084) +- Refactor disable preview feature to run entirely on the frontend without backend calls. (#2067) +- Implement a method to send JSON data to the VSCode integration without running the Kedro-Viz server. (#2049) + + # Release 9.2.0 ## Major features and improvements diff --git a/cypress/fixtures/mock/compatibleMetadata.json b/cypress/fixtures/mock/compatibleMetadata.json new file mode 100644 index 0000000000..7aca2c11bf --- /dev/null +++ b/cypress/fixtures/mock/compatibleMetadata.json @@ -0,0 +1,15 @@ +{ + "has_missing_dependencies": false, + "package_compatibilities": [ + { + "package_name": "fsspec", + "package_version": "2023.9.1", + "is_compatible": true + }, + { + "package_name": "kedro-datasets", + "package_version": "2.0.0", + "is_compatible": true + } + ] +} diff --git a/cypress/fixtures/mock/package-compatibilities-incompatible.json b/cypress/fixtures/mock/inCompatibleMetadata.json similarity index 66% rename from cypress/fixtures/mock/package-compatibilities-incompatible.json rename to cypress/fixtures/mock/inCompatibleMetadata.json index d33d26f125..109ca1bfa7 100644 --- a/cypress/fixtures/mock/package-compatibilities-incompatible.json +++ b/cypress/fixtures/mock/inCompatibleMetadata.json @@ -1,12 +1,15 @@ -[ - { +{ + "has_missing_dependencies": true, + "package_compatibilities": [ + { "package_name": "fsspec", "package_version": "2023.8.1", "is_compatible": false - }, - { + }, + { "package_name": "kedro-datasets", "package_version": "1.8.0", "is_compatible": false - } -] + } + ] +} diff --git a/cypress/fixtures/mock/package-compatibilities-compatible.json b/cypress/fixtures/mock/package-compatibilities-compatible.json deleted file mode 100644 index 4480b400e9..0000000000 --- a/cypress/fixtures/mock/package-compatibilities-compatible.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "package_name": "fsspec", - "package_version": "2023.9.1", - "is_compatible": true - }, - { - "package_name": "kedro-datasets", - "package_version": "1.8.0", - "is_compatible": false - } -] \ No newline at end of file diff --git a/cypress/tests/ui/flowchart/banners.cy.js b/cypress/tests/ui/flowchart/banners.cy.js new file mode 100644 index 0000000000..bdeb20bb4b --- /dev/null +++ b/cypress/tests/ui/flowchart/banners.cy.js @@ -0,0 +1,32 @@ +describe('Banners in Kedro-Viz', () => { + beforeEach(() => { + // Clears localStorage before each test + cy.clearLocalStorage(); + }); + + it("shows a missing dependencies banner in viz lite mode if the kedro project dependencies are not installed.", () => { + // Intercept the network request to mock with a fixture + cy.__interceptRest__( + '/api/metadata', + 'GET', + '/mock/inCompatibleMetadata.json' + ).as("appMetadata"); + + // Action + cy.reload(); + + // Assert after action + cy.get('[data-test="flowchart-wrapper--lite-banner"]').should('exist'); + cy.get('.banner-message-body').should('contains.text', 'please install the missing Kedro project dependencies') + cy.get('.banner-message-title').should('contains.text', 'Missing dependencies') + + // Test Learn more link + cy.get(".banner a") + .should("contains.attr", "href", "https://docs.kedro.org/projects/kedro-viz/en/latest/kedro-viz_visualisation.html#visualise-a-kedro-project-without-installing-project-dependencies"); + + // Close the banner + cy.get(".banner-close").click() + cy.get('[data-test="flowchart-wrapper--lite-banner"]').should('not.exist'); + + }); +}); diff --git a/cypress/tests/ui/flowchart/flowchart.cy.js b/cypress/tests/ui/flowchart/flowchart.cy.js index 39b4a37128..a2eee6fbde 100644 --- a/cypress/tests/ui/flowchart/flowchart.cy.js +++ b/cypress/tests/ui/flowchart/flowchart.cy.js @@ -6,13 +6,14 @@ describe('Flowchart DAG', () => { beforeEach(() => { cy.enablePrettyNames(); // Enable pretty names using the custom command + cy.wait(500); + cy.get('.feature-hints__close').click(); // Close the feature hints so can click on a node + cy.wait(500); }); it('verifies that users can expand a collapsed modular pipeline in the flowchart. #TC-23', () => { const modularPipelineText = 'feature_engineering'; - const taskNodeText = 'Create Derived Features'; - - cy.enablePrettyNames(); + const taskNodeText = 'create_derived_features'; // Assert before action cy.get('.pipeline-node > .pipeline-node__text').should( diff --git a/cypress/tests/ui/flowchart/menu.cy.js b/cypress/tests/ui/flowchart/menu.cy.js index 833c081efe..deb6d38f81 100644 --- a/cypress/tests/ui/flowchart/menu.cy.js +++ b/cypress/tests/ui/flowchart/menu.cy.js @@ -1,10 +1,11 @@ // All E2E Tests Related to Flowchart Menu goes here. -import { prettifyName } from '../../../../src/utils'; - describe('Flowchart Menu', () => { beforeEach(() => { cy.enablePrettyNames(); // Enable pretty names using the custom command + cy.wait(500); + cy.get('.feature-hints__close').click(); // Close the feature hints so can click on a node + cy.wait(500); }); it('verifies that users can select a section of the flowchart through the drop down. #TC-16', () => { @@ -144,7 +145,7 @@ describe('Flowchart Menu', () => { .invoke('text') .then((focusedNodesText) => expect(focusedNodesText.toLowerCase()).to.contains( - prettifyName(nodeToFocusText).toLowerCase() + nodeToFocusText ) ); cy.get('.pipeline-node--active > .pipeline-node__text').should( diff --git a/cypress/tests/ui/flowchart/panel.cy.js b/cypress/tests/ui/flowchart/panel.cy.js index 9f525d99a2..db246d382d 100644 --- a/cypress/tests/ui/flowchart/panel.cy.js +++ b/cypress/tests/ui/flowchart/panel.cy.js @@ -124,7 +124,7 @@ describe('Pipeline Minimap Toolbar', () => { cy.__waitForPageLoad__(() => { let initialZoomValue; let zoomInValue; - + cy.get('@zoomScale') .invoke('text') .then((text) => { @@ -155,6 +155,7 @@ describe('Pipeline Minimap Toolbar', () => { cy.get('@zoomScale') .invoke('text') .should((text) => { + initialZoomValue = parseFloat(text.replace('%', '')); expect(initialZoomValue).to.be.eq(parseFloat(text.replace('%', ''))); }); }); diff --git a/cypress/tests/ui/flowchart/shareable-urls.cy.js b/cypress/tests/ui/flowchart/shareable-urls.cy.js index a02f31c5bd..776e745ffc 100644 --- a/cypress/tests/ui/flowchart/shareable-urls.cy.js +++ b/cypress/tests/ui/flowchart/shareable-urls.cy.js @@ -7,9 +7,9 @@ describe('Shareable URLs with empty localStorage', () => { it('verifies that users can open the Deploy Kedro-Viz modal if the localStorage is empty. #TC-52', () => { // Intercept the network request to mock with a fixture cy.__interceptRest__( - '/api/package-compatibilities', + '/api/metadata', 'GET', - '/mock/package-compatibilities-compatible.json' + '/mock/compatibleMetadata.json' ); // Action @@ -25,9 +25,9 @@ describe('Shareable URLs with empty localStorage', () => { it("shows an incompatible message given the user's fsspec package version is outdated. #TC-53", () => { // Intercept the network request to mock with a fixture cy.__interceptRest__( - '/api/package-compatibilities', + '/api/metadata', 'GET', - '/mock/package-compatibilities-incompatible.json' + '/mock/inCompatibleMetadata.json' ); // Action diff --git a/demo-project/.version b/demo-project/.version index deeb3d66ef..a13e7b9c87 100755 --- a/demo-project/.version +++ b/demo-project/.version @@ -1 +1 @@ -9.2.0 +10.0.0 diff --git a/demo-project/build/api/metadata b/demo-project/build/api/metadata new file mode 100644 index 0000000000..52afe268cb --- /dev/null +++ b/demo-project/build/api/metadata @@ -0,0 +1,15 @@ +{ + "has_missing_dependencies": false, + "package_compatibilities": [ + { + "package_name": "fsspec", + "package_version": "2024.6.1", + "is_compatible": true + }, + { + "package_name": "kedro-datasets", + "package_version": "4.0.0", + "is_compatible": true + } + ] +} diff --git a/demo-project/build/api/package-compatibilities b/demo-project/build/api/package-compatibilities deleted file mode 100644 index d77cf5c71d..0000000000 --- a/demo-project/build/api/package-compatibilities +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "package_name": "fsspec", - "package_version": "2024.6.1", - "is_compatible": true - }, - { - "package_name": "kedro-datasets", - "package_version": "4.0.0", - "is_compatible": true - } -] \ No newline at end of file diff --git a/demo-project/lightsail.json b/demo-project/lightsail.json index ec9d3fed2d..b5bc4eabcc 100644 --- a/demo-project/lightsail.json +++ b/demo-project/lightsail.json @@ -2,7 +2,7 @@ "serviceName": "kedro-viz-live-demo", "containers": { "kedro-viz-live-demo": { - "image": "public.ecr.aws/g0x0s3o2/kedro-viz-live-demo:9.2.0", + "image": "public.ecr.aws/g0x0s3o2/kedro-viz-live-demo:10.0.0", "ports": { "4141": "HTTP" } diff --git a/docs/source/cli-docs.md b/docs/source/cli-docs.md new file mode 100644 index 0000000000..72ea1b8af8 --- /dev/null +++ b/docs/source/cli-docs.md @@ -0,0 +1,165 @@ +# Kedro Viz CLI reference + +The Kedro Viz CLI provides commands to visualise Kedro pipelines, deploy them to cloud platforms, and export the visualisation data. Below is a detailed description of the available commands and options. + +## Commands + +### `kedro viz` + +Launches a local Kedro Viz instance to visualise a Kedro pipeline. + +**Usage:** + +```bash +kedro viz [OPTIONS] +``` + +**Description:** + +This command launches the Kedro Viz server to visualise a Kedro pipeline. It is functionally the same as `kedro viz run`. If no sub-command is provided, `run` is used by default. + +**Options:** + +This command accepts all the options that are available in the `kedro viz`, `kedro viz run` command. See the `kedro viz run` section for a complete list of options. + +### `kedro viz run` + +Launches a local Kedro Viz instance to visualise a Kedro pipeline. + +**Usage:** + +```bash +kedro viz run [OPTIONS] +``` + +**Options:** + +- `--host ` + - Host that Kedro Viz will listen to. Defaults to `localhost`. + +- `--port ` + - TCP port that Kedro Viz will listen to. Defaults to `4141`. + +- `--browser / --no-browser` + - Whether to open the Kedro Viz interface in the default browser. The browser will open if the host is `localhost`. Defaults to `True`. + +- `--load-file ` + - Path to load Kedro Viz data from a directory. If provided, Kedro Viz will load the visualisation data from this path instead of generating it from the pipeline. + +- `--save-file ` + - Path to save Kedro Viz data to a directory. If provided, the visualisation data will be saved to this path for later use. + +- `--pipeline, -p ` + - Name of the registered pipeline to visualise. If not set, the default pipeline is visualised. + +- `--env, -e {environment>}` + - Kedro configuration environment. If not specified, the catalog config in `local` will be used. You can also set this through the `KEDRO_ENV` environment variable. + +- `--autoreload, -a` + - Enable autoreload of the Kedro Viz server when a Python or YAML file changes in the Kedro project. + +- `--include-hooks` + - Include all registered hooks in the Kedro project for visualisation. + +- `--params ` + - Specify extra parameters for the Kedro Viz run. This option supports the same format as the `params` option in the Kedro CLI. + +- `--lite` + - An experimental flag to open Kedro-Viz without Kedro project dependencies. + + +```{note} +When running Kedro Viz locally with the `--autoreload` option, the server will automatically restart whenever there are changes to Python, YAML, or JSON files in the Kedro project. This is particularly useful during development. +``` + + +### `kedro viz deploy` + +Deploy and host Kedro Viz on a specified cloud platform. + +```{note} +The `deploy` command supports deployment to AWS, Azure and GCP. Ensure that your cloud credentials and configurations are correctly set up before deploying. +``` + +**Usage:** + +```bash +kedro viz deploy [OPTIONS] +``` + +**Options:** + +- `--platform ` + - The cloud platform to host Kedro Viz on. Supported platforms include `aws` `azure` and `gcp`. This option is required. + +- `--endpoint ` + - The static website hosted endpoint. This option is required. + +- `--bucket-name ` + - The name of the bucket where Kedro Viz will be hosted. This option is required. + +- `--include-hooks` + - Include all registered hooks in the Kedro project in the deployed visualisation. + +- `--include-previews` + - Include previews for all datasets in the deployed visualisation. + +### `kedro viz build` + +Create a build directory of a local Kedro Viz instance with Kedro project data. + +**Usage:** + +```bash +kedro viz build [OPTIONS] +``` + +**Options:** + +- `--include-hooks` + - Include all registered hooks in the Kedro project in the built visualisation. + +- `--include-previews` + - Include previews for all datasets in the built visualisation. + + +## Examples + +### Running Kedro Viz locally + +To run Kedro Viz on your local machine, use: + +```bash +kedro viz +``` + +To specify a particular pipeline and environment: + +```bash +kedro viz -p my_pipeline -e dev +``` + +or + +```bash +kedro viz run -p my_pipeline -e dev +``` + +### Deploying Kedro Viz to AWS + +To deploy Kedro Viz to an S3 bucket on AWS: + +```bash +kedro viz deploy --platform aws --endpoint http://mybucket.s3-website-us-west-2.amazonaws.com --bucket-name mybucket +``` + +### Building Kedro Viz to host on multiple platforms + +To create a build directory with the visualisation data: + +```bash +kedro viz build --include-previews +``` + + + diff --git a/docs/source/experiment_tracking.md b/docs/source/experiment_tracking.md index 0c3870ecd4..c06ca5a208 100644 --- a/docs/source/experiment_tracking.md +++ b/docs/source/experiment_tracking.md @@ -346,7 +346,7 @@ Parallel coordinates displays all metrics on a single graph, with each vertical When in comparison view, comparing runs highlights your selections on the respective chart types, improving readability even in the event there is a multitude of data points. ```{note} -The following graphic is taken from the [Kedro-Viz experiment tracking demo](https://demo.kedro.org/experiment-tracking) (it is not a visualisation from the example code you created above). +The following graphic is taken from the [Kedro-Viz experiment tracking demo](https://demo.kedro.org/) (it is not a visualisation from the example code you created above). ``` ![](./images/experiment-tracking-metrics-comparison.gif) diff --git a/docs/source/images/preview_datasets_json.png b/docs/source/images/preview_datasets_json.png new file mode 100644 index 0000000000..7b160e9101 Binary files /dev/null and b/docs/source/images/preview_datasets_json.png differ diff --git a/docs/source/images/slice_pipeline_multiple_click.gif b/docs/source/images/slice_pipeline_multiple_click.gif new file mode 100644 index 0000000000..fa6a6448fa Binary files /dev/null and b/docs/source/images/slice_pipeline_multiple_click.gif differ diff --git a/docs/source/images/slice_pipeline_slice_reset.gif b/docs/source/images/slice_pipeline_slice_reset.gif new file mode 100644 index 0000000000..8185ad2a5d Binary files /dev/null and b/docs/source/images/slice_pipeline_slice_reset.gif differ diff --git a/docs/source/images/viz-in-vscode.gif b/docs/source/images/viz-in-vscode.gif new file mode 100644 index 0000000000..cded1693f9 Binary files /dev/null and b/docs/source/images/viz-in-vscode.gif differ diff --git a/docs/source/index.md b/docs/source/index.md index cfc7302bcb..ea10570cfb 100755 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -29,5 +29,13 @@ Take a look at the TablePreview: - filtered_data = self.data + data = self.load() for column, value in filters.items(): - filtered_data = filtered_data[filtered_data[column] == value] - subset = filtered_data.iloc[:nrows, :ncolumns] - df_dict = {} - for column in subset.columns: - df_dict[column] = subset[column] - return df_dict - + data = data[data[column] == value] + subset = data.iloc[:nrows, :ncolumns] + preview_data = { + 'index': list(subset.index), # List of row indices + 'columns': list(subset.columns), # List of column names + 'data': subset.values.tolist() # List of rows, where each row is a list of values + } + return preview_data ``` +![](./images/preview_datasets_expanded.png) -## Examples of Previews +## ImagePreview +For `ImagePreview`, the function should return a base64-encoded string representing the image. This is typically used for datasets that output visual data such as plots or images. -1. TablePreview +Below is an example implementation: -![](./images/preview_datasets_expanded.png) +```python +from kedro_datasets._typing import ImagePreview -2. ImagePreview +class CustomImageDataset: + def preview(self) -> ImagePreview: + image_path = self._get_image_path() + with open(image_path, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode('utf-8') + return ImagePreview(encoded_string) +``` ![](./images/pipeline_visualisation_matplotlib_expand.png) +## PlotlyPreview +For `PlotlyPreview`, the function should return a dictionary containing Plotly figure data. This includes the figure's `data` and `layout` keys. + +Below is an example implementation: -3. PlotlyPreview +```python + +from kedro_datasets._typing import PlotlyPreview + +class CustomPlotlyDataset: + def preview(self) -> PlotlyPreview: + figure = self._load_plotly_figure() + return PlotlyPreview({ + "data": figure["data"], + "layout": figure["layout"] + }) +``` ![](./images/pipeline_visualisation_plotly_expand_1.png) +## JSONPreview +For `JSONPreview`, the function should return a dictionary representing the `JSON` data. This is useful for previewing complex nested data structures. +Below is an example implementation: +```python + +from kedro_datasets._typing import JSONPreview +class CustomJSONDataset: + def preview(self) -> JSONPreview: + json_data = self._load_json_data() + return JSONPreview(json.dumps(json_data)) +``` +![](./images/preview_datasets_json.png) diff --git a/docs/source/preview_datasets.md b/docs/source/preview_datasets.md index 474822e1f6..1d6b455e22 100644 --- a/docs/source/preview_datasets.md +++ b/docs/source/preview_datasets.md @@ -57,6 +57,8 @@ companies: preview: false ``` +You can also disable previews globally through the settings menu on Kedro-Viz. + ```{note} Starting from Kedro-Viz 9.2.0, previews are disabled by default for the CLI commands `kedro viz deploy` and `kedro viz build`. You can control this behavior using the `--include-previews` flag with these commands. For `kedro viz run`, previews are enabled by default and can be controlled from the publish modal dialog, refer to the [Publish and share](./share_kedro_viz) for more instructions. ``` \ No newline at end of file diff --git a/docs/source/slice_a_pipeline.md b/docs/source/slice_a_pipeline.md new file mode 100644 index 0000000000..d3b0eee2ba --- /dev/null +++ b/docs/source/slice_a_pipeline.md @@ -0,0 +1,33 @@ +# Slice a pipeline + +Slicing a pipeline in Kedro refers to creating a subset of a pipeline's nodes, which can help in focusing on specific parts of the pipeline. There are two primary ways to achieve this: + +1. **Programmatically with the Kedro CLI.** This method is suitable for those comfortable with command-line tools. Detailed steps on how to achieve this are available in the kedro documentation: [Slice a Pipeline](https://docs.kedro.org/en/stable/nodes_and_pipelines/slice_a_pipeline.html). + +2. **Visually through Kedro-Viz:** This approach allows you to visually select and slice pipeline nodes, which then generates a run command for executing the slice within your Kedro project. + +## Benefits of Kedro-Viz slicing + +- **Visual Representation:** View the relationships between nodes and identify which ones are part of your slice. +- **Immediate Command Generation:** Get a ready-to-use CLI command for executing the sliced pipeline. +- **Interactive Control:** Visually select and reset slices with a couple of clicks. + +## Steps to slice in Kedro-Viz + +Kedro-Viz offers a user-friendly visual interface for slicing pipelines. Follow these steps to use the slicing feature: + +1. **Select elements in the flowchart:** In Kedro-Viz, select two elements to set the boundaries for your slice: + - Click on the first node you want to include. + - Hold the Shift key and select the second node. + +![](./images/slice_pipeline_multiple_click.gif) + +2. **Highlighted selection:** The flowchart will highlight all nodes between the selected elements, and the corresponding nodes in the list on the left will also be highlighted. + +3. **View the run command:** After selecting the nodes, Kedro-Viz generates a CLI command for the sliced pipeline. You can copy this command and use it directly in your Kedro project to run the slice. + +4. **Slice the pipeline:** When you're ready, click the "Slice" button. This opens a new view where you can directly interact with the sliced pipeline. + +5. **Reset:** To discard your selection and return to the full pipeline view, click the "Reset" button. This will clear the slice and restore the default view. + +![](./images/slice_pipeline_slice_reset.gif) diff --git a/package-lock.json b/package-lock.json index 8c3388886d..790cb4a30e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@quantumblack/kedro-viz", - "version": "9.2.0", + "version": "10.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@quantumblack/kedro-viz", - "version": "9.1.0", + "version": "9.2.0", "dependencies": { "@apollo/client": "^3.5.6", "@emotion/react": "^11.10.6", @@ -9207,9 +9207,9 @@ "dev": true }, "node_modules/body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", "dev": true, "dependencies": { "bytes": "3.1.2", @@ -9220,7 +9220,7 @@ "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.11.0", + "qs": "6.13.0", "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" @@ -9267,12 +9267,12 @@ "dev": true }, "node_modules/body-parser/node_modules/qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "dev": true, "dependencies": { - "side-channel": "^1.0.4" + "side-channel": "^1.0.6" }, "engines": { "node": ">=0.6" @@ -14118,37 +14118,37 @@ } }, "node_modules/express": { - "version": "4.19.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", - "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", + "version": "4.20.0", + "resolved": "https://registry.npmjs.org/express/-/express-4.20.0.tgz", + "integrity": "sha512-pLdae7I6QqShF5PnNTCVn4hI91Dx0Grkn2+IAsMTgMIKuQVte2dN9PeGSSAME2FR8anOhVA62QDIUaWVfEXVLw==", "dev": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.2", + "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "1.2.0", "fresh": "0.5.2", "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", + "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", + "path-to-regexp": "0.1.10", "proxy-addr": "~2.0.7", "qs": "6.11.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", + "send": "0.19.0", + "serve-static": "1.16.0", "setprototypeof": "1.2.0", "statuses": "2.0.1", "type-is": "~1.6.18", @@ -14168,6 +14168,15 @@ "ms": "2.0.0" } }, + "node_modules/express/node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/express/node_modules/ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -14175,9 +14184,9 @@ "dev": true }, "node_modules/express/node_modules/path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==", + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", + "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==", "dev": true }, "node_modules/express/node_modules/qs": { @@ -23672,10 +23681,13 @@ } }, "node_modules/merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==", - "dev": true + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/merge-stream": { "version": "2.0.0", @@ -30257,9 +30269,9 @@ } }, "node_modules/send": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", "dev": true, "dependencies": { "debug": "2.6.9", @@ -30416,9 +30428,9 @@ } }, "node_modules/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.0.tgz", + "integrity": "sha512-pDLK8zwl2eKaYrs8mrPZBJua4hMplRWJ1tIFksVC3FtBEBnl8dxgeHtsaMS8DhS9i4fLObaon6ABoc4/hQGdPA==", "dev": true, "dependencies": { "encodeurl": "~1.0.2", @@ -30430,6 +30442,51 @@ "node": ">= 0.8.0" } }, + "node_modules/serve-static/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/serve-static/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/serve-static/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "node_modules/serve-static/node_modules/send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", @@ -34759,4 +34816,4 @@ } } } -} +} \ No newline at end of file diff --git a/package.json b/package.json index fcfbf0c864..5883aa9594 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@quantumblack/kedro-viz", - "version": "9.2.0", + "version": "10.0.0", "description": "Kedro-Viz is an interactive development tool for building data science pipelines with Kedro.", "main": "lib/components/app/index.js", "files": [ @@ -172,4 +172,4 @@ "not op_mini all" ], "snyk": true -} \ No newline at end of file +} diff --git a/package/README.md b/package/README.md index 3b19bc1e35..cd8998d2e9 100644 --- a/package/README.md +++ b/package/README.md @@ -157,6 +157,9 @@ Options: To pass a nested dictionary as parameter, separate keys by '.', example: param_group.param1:value1. + --lite An experimental flag to open Kedro-Viz without Kedro + project dependencies. + -h, --help Show this message and exit. ``` @@ -272,6 +275,12 @@ For more information on how to use Kedro-Viz as a React component, follow this [ **_Our documentation contains [additional examples on how to visualise with Kedro-Viz.](https://docs.kedro.org/en/stable/visualisation/index.html)_** +## Kedro-Viz in Visual Studio Code Extension + +To visualize Kedro project using Kedro-Viz in Visual Studio Code, please install Kedro extension in Visual Studio Code. + +For more information on how to use Kedro-Viz in Visual Studio Code, follow this [guide](https://marketplace.visualstudio.com/items?itemName=kedro.Kedro) + ## Feature Flags Kedro-Viz uses features flags to roll out some experimental features. The following flags are currently in use: diff --git a/package/features/steps/cli_steps.py b/package/features/steps/cli_steps.py index c452217521..769cb08d64 100644 --- a/package/features/steps/cli_steps.py +++ b/package/features/steps/cli_steps.py @@ -147,6 +147,16 @@ def exec_viz_command(context): ) +@when("I execute the kedro viz run command with lite option") +def exec_viz_lite_command(context): + """Execute Kedro-Viz command.""" + context.result = ChildTerminatingPopen( + [context.kedro, "viz", "run", "--lite", "--no-browser"], + env=context.env, + cwd=str(context.root_project_dir), + ) + + @then("kedro-viz should start successfully") def check_kedroviz_up(context): """Check that Kedro-Viz is up and responding to requests.""" @@ -169,3 +179,26 @@ def check_kedroviz_up(context): ) finally: context.result.terminate() + + +@then("I store the response from main endpoint") +def get_main_api_response(context): + max_duration = 30 # 30 seconds + end_by = time() + max_duration + + while time() < end_by: + try: + response = requests.get("http://localhost:4141/api/main") + context.response = response.json() + assert response.status_code == 200 + except Exception: + sleep(2.0) + continue + else: + break + + +@then("I compare the responses in regular and lite mode") +def compare_main_api_responses(context): + regular_mode_response = requests.get("http://localhost:4141/api/main").json() + assert context.response == regular_mode_response diff --git a/package/features/viz.feature b/package/features/viz.feature index 75c7b65fed..d3c01e2f7f 100644 --- a/package/features/viz.feature +++ b/package/features/viz.feature @@ -24,3 +24,17 @@ Feature: Viz plugin in new project When I execute the kedro viz run command Then kedro-viz should start successfully + Scenario: Execute viz lite with latest Kedro + Given I have installed kedro version "latest" + And I have run a non-interactive kedro new with spaceflights-pandas starter + When I execute the kedro viz run command with lite option + Then kedro-viz should start successfully + + Scenario: Compare viz responses in regular and lite mode + Given I have installed kedro version "latest" + And I have run a non-interactive kedro new with spaceflights-pandas starter + When I execute the kedro viz run command with lite option + Then I store the response from main endpoint + Given I have installed the project's requirements + When I execute the kedro viz run command + Then I compare the responses in regular and lite mode diff --git a/package/kedro_viz/__init__.py b/package/kedro_viz/__init__.py index b20a6ce491..6053ab4eac 100644 --- a/package/kedro_viz/__init__.py +++ b/package/kedro_viz/__init__.py @@ -2,7 +2,7 @@ import sys import warnings -__version__ = "9.2.0" +__version__ = "10.0.0" class KedroVizPythonVersionWarning(UserWarning): diff --git a/package/kedro_viz/api/rest/requests.py b/package/kedro_viz/api/rest/requests.py index b35f4f39b0..6f0a8bafb3 100644 --- a/package/kedro_viz/api/rest/requests.py +++ b/package/kedro_viz/api/rest/requests.py @@ -10,9 +10,3 @@ class DeployerConfiguration(BaseModel): is_all_previews_enabled: bool = False endpoint: str bucket_name: str - - -class UserPreference(BaseModel): - """User preferences for Kedro Viz.""" - - showDatasetPreviews: bool diff --git a/package/kedro_viz/api/rest/responses.py b/package/kedro_viz/api/rest/responses.py index 8da71d0354..2f59d33b16 100644 --- a/package/kedro_viz/api/rest/responses.py +++ b/package/kedro_viz/api/rest/responses.py @@ -2,17 +2,16 @@ # pylint: disable=missing-class-docstring,invalid-name import abc +import json import logging -from importlib.metadata import PackageNotFoundError from typing import Any, Dict, List, Optional, Union import orjson -import packaging from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse, ORJSONResponse from pydantic import BaseModel, ConfigDict -from kedro_viz.api.rest.utils import get_package_version +from kedro_viz.api.rest.utils import get_package_compatibilities from kedro_viz.data_access import data_access_manager from kedro_viz.models.flowchart import ( DataNode, @@ -23,6 +22,7 @@ TranscodedDataNode, TranscodedDataNodeMetadata, ) +from kedro_viz.models.metadata import Metadata, PackageCompatibility logger = logging.getLogger(__name__) @@ -143,7 +143,7 @@ class DataNodeMetadataAPIResponse(BaseAPIResponse): class TranscodedDataNodeMetadataAPIReponse(BaseAPIResponse): - filepath: str + filepath: Optional[str] = None original_type: str transcoded_types: List[str] run_command: Optional[str] = None @@ -258,17 +258,24 @@ class GraphAPIResponse(BaseAPIResponse): selected_pipeline: str -class PackageCompatibilityAPIResponse(BaseAPIResponse): - package_name: str - package_version: str - is_compatible: bool +class MetadataAPIResponse(BaseAPIResponse): + has_missing_dependencies: bool = False + package_compatibilities: List[PackageCompatibility] = [] model_config = ConfigDict( json_schema_extra={ - "example": { - "package_name": "fsspec", - "package_version": "2023.9.1", - "is_compatible": True, - } + "has_missing_dependencies": False, + "package_compatibilities": [ + { + "package_name": "fsspec", + "package_version": "2024.6.1", + "is_compatible": True, + }, + { + "package_name": "kedro-datasets", + "package_version": "4.0.0", + "is_compatible": True, + }, + ], } ) @@ -371,46 +378,49 @@ def get_selected_pipeline_response(registered_pipeline_id: str): ) -def get_package_compatibilities_response( - package_requirements: Dict[str, Dict[str, str]], -) -> List[PackageCompatibilityAPIResponse]: - """API response for `/api/package_compatibility`.""" - package_requirements_response = [] - - for package_name, package_info in package_requirements.items(): - compatible_version = package_info["min_compatible_version"] - try: - package_version = get_package_version(package_name) - except PackageNotFoundError: - logger.warning(package_info["warning_message"]) - package_version = "0.0.0" - - is_compatible = packaging.version.parse( - package_version - ) >= packaging.version.parse(compatible_version) - - package_requirements_response.append( - PackageCompatibilityAPIResponse( - package_name=package_name, - package_version=package_version, - is_compatible=is_compatible, - ) - ) - - return package_requirements_response +def get_metadata_response(): + """API response for `/api/metadata`.""" + package_compatibilities = get_package_compatibilities() + Metadata.set_package_compatibilities(package_compatibilities) + return Metadata() -def write_api_response_to_fs(file_path: str, response: Any, remote_fs: Any): - """Encodes, enhances responses and writes it to a file""" +def get_encoded_response(response: Any) -> bytes: + """Encodes and enhances the default response using human-readable format.""" jsonable_response = jsonable_encoder(response) encoded_response = EnhancedORJSONResponse.encode_to_human_readable( jsonable_response ) + return encoded_response + + +def write_api_response_to_fs(file_path: str, response: Any, remote_fs: Any): + """Get encoded responses and writes it to a file""" + encoded_response = get_encoded_response(response) + with remote_fs.open(file_path, "wb") as file: file.write(encoded_response) +def get_kedro_project_json_data(): + """Decodes the default response and returns the Kedro project JSON data. + This will be used in VSCode extension to get current Kedro project data.""" + encoded_response = get_encoded_response(get_default_response()) + + try: + response_str = encoded_response.decode("utf-8") + json_data = json.loads(response_str) + except UnicodeDecodeError as exc: # pragma: no cover + json_data = None + logger.error("Failed to decode response string. Error: %s", str(exc)) + except json.JSONDecodeError as exc: # pragma: no cover + json_data = None + logger.error("Failed to parse JSON data. Error: %s", str(exc)) + + return json_data + + def save_api_main_response_to_fs(main_path: str, remote_fs: Any): """Saves API /main response to a directory.""" try: diff --git a/package/kedro_viz/api/rest/router.py b/package/kedro_viz/api/rest/router.py index 64ad2dc4d6..3cd6a18e9f 100644 --- a/package/kedro_viz/api/rest/router.py +++ b/package/kedro_viz/api/rest/router.py @@ -2,24 +2,21 @@ # pylint: disable=missing-function-docstring, broad-exception-caught import logging -from typing import List from fastapi import APIRouter from fastapi.responses import JSONResponse -from kedro_viz.api.rest.requests import DeployerConfiguration, UserPreference -from kedro_viz.constants import PACKAGE_REQUIREMENTS +from kedro_viz.api.rest.requests import DeployerConfiguration from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory from .responses import ( APIErrorMessage, - DataNodeMetadata, GraphAPIResponse, + MetadataAPIResponse, NodeMetadataAPIResponse, - PackageCompatibilityAPIResponse, get_default_response, + get_metadata_response, get_node_metadata_response, - get_package_compatibilities_response, get_selected_pipeline_response, ) @@ -50,36 +47,6 @@ async def get_single_node_metadata(node_id: str): return get_node_metadata_response(node_id) -@router.post("/preferences") -async def update_preferences(preferences: UserPreference): - try: - DataNodeMetadata.set_is_all_previews_enabled(preferences.showDatasetPreviews) - return JSONResponse( - status_code=200, content={"message": "Preferences updated successfully"} - ) - except Exception as exception: - logger.error("Failed to update preferences: %s", str(exception)) - return JSONResponse( - status_code=500, - content={"message": "Failed to update preferences"}, - ) - - -@router.get("/preferences", response_model=UserPreference) -async def get_preferences(): - try: - show_dataset_previews = DataNodeMetadata.is_all_previews_enabled - return JSONResponse( - status_code=200, content={"showDatasetPreviews": show_dataset_previews} - ) - except Exception as exception: - logger.error("Failed to fetch preferences: %s", str(exception)) - return JSONResponse( - status_code=500, - content={"message": "Failed to fetch preferences"}, - ) - - @router.get( "/pipelines/{registered_pipeline_id}", response_model=GraphAPIResponse, @@ -122,17 +89,15 @@ async def deploy_kedro_viz(input_values: DeployerConfiguration): @router.get( - "/package-compatibilities", - response_model=List[PackageCompatibilityAPIResponse], + "/metadata", + response_model=MetadataAPIResponse, ) -async def get_package_compatibilities(): +async def get_metadata(): try: - return get_package_compatibilities_response(PACKAGE_REQUIREMENTS) + return get_metadata_response() except Exception as exc: - logger.exception( - "An exception occurred while getting package compatibility info : %s", exc - ) + logger.exception("An exception occurred while getting app metadata: %s", exc) return JSONResponse( status_code=500, - content={"message": "Failed to get package compatibility info"}, + content={"message": "Failed to get app metadata"}, ) diff --git a/package/kedro_viz/api/rest/utils.py b/package/kedro_viz/api/rest/utils.py index 87ed619633..dd0ba584d1 100644 --- a/package/kedro_viz/api/rest/utils.py +++ b/package/kedro_viz/api/rest/utils.py @@ -1,10 +1,49 @@ """`kedro_viz.api.rest.utils` contains utility functions used in the `kedro_viz.api.rest` package""" + +import logging +from importlib.metadata import PackageNotFoundError +from typing import List + +import packaging + +from kedro_viz.constants import PACKAGE_REQUIREMENTS +from kedro_viz.models.metadata import PackageCompatibility + try: from importlib.metadata import version except ImportError: # pragma: no cover from importlib_metadata import version +logger = logging.getLogger(__name__) + def get_package_version(package_name: str): """Returns the version of the given package.""" return version(package_name) # pragma: no cover + + +def get_package_compatibilities() -> List[PackageCompatibility]: + """Returns the package compatibilities information + for the current python env.""" + package_compatibilities: List[PackageCompatibility] = [] + + for package_name, package_info in PACKAGE_REQUIREMENTS.items(): + compatible_version = package_info["min_compatible_version"] + try: + package_version = get_package_version(package_name) + except PackageNotFoundError: + logger.warning(package_info["warning_message"]) + package_version = "0.0.0" + + is_compatible = packaging.version.parse( + package_version + ) >= packaging.version.parse(compatible_version) + + package_compatibilities.append( + PackageCompatibility( + package_name=package_name, + package_version=package_version, + is_compatible=is_compatible, + ) + ) + return package_compatibilities diff --git a/package/kedro_viz/data_access/managers.py b/package/kedro_viz/data_access/managers.py index 4eb3e72130..9801c86cb7 100644 --- a/package/kedro_viz/data_access/managers.py +++ b/package/kedro_viz/data_access/managers.py @@ -7,11 +7,13 @@ import networkx as nx from kedro.io import DataCatalog +from kedro.io.core import DatasetError from kedro.pipeline import Pipeline as KedroPipeline from kedro.pipeline.node import Node as KedroNode from sqlalchemy.orm import sessionmaker from kedro_viz.constants import DEFAULT_REGISTERED_PIPELINE_ID, ROOT_MODULAR_PIPELINE_ID +from kedro_viz.integrations.utils import UnavailableDataset from kedro_viz.models.flowchart import ( DataNode, GraphEdge, @@ -316,7 +318,15 @@ def add_dataset( Returns: The GraphNode instance representing the dataset that was added to the NodesRepository. """ - obj = self.catalog.get_dataset(dataset_name) + try: + obj = self.catalog.get_dataset(dataset_name) + except DatasetError: + # This is to handle dataset factory patterns when running + # Kedro Viz in lite mode. The `get_dataset` function + # of DataCatalog calls AbstractDataset.from_config + # which tries to create a Dataset instance from the pattern + obj = UnavailableDataset() + layer = self.catalog.get_layer_for_dataset(dataset_name) graph_node: Union[DataNode, TranscodedDataNode, ParametersNode] ( diff --git a/package/kedro_viz/integrations/kedro/abstract_dataset_lite.py b/package/kedro_viz/integrations/kedro/abstract_dataset_lite.py new file mode 100644 index 0000000000..582130de00 --- /dev/null +++ b/package/kedro_viz/integrations/kedro/abstract_dataset_lite.py @@ -0,0 +1,32 @@ +"""``AbstractDatasetLite`` is a custom implementation of Kedro's ``AbstractDataset`` +to provide an UnavailableDataset instance when running Kedro-Viz in lite mode. +""" + +import logging +from typing import Any, Optional + +from kedro.io.core import AbstractDataset, DatasetError + +from kedro_viz.integrations.utils import UnavailableDataset + +logger = logging.getLogger(__name__) + + +class AbstractDatasetLite(AbstractDataset): + """``AbstractDatasetLite`` is a custom implementation of Kedro's ``AbstractDataset`` + to provide an UnavailableDataset instance by overriding ``from_config`` of ``AbstractDataset`` + when running Kedro-Viz in lite mode. + """ + + @classmethod + def from_config( + cls: type, + name: str, + config: dict[str, Any], + load_version: Optional[str] = None, + save_version: Optional[str] = None, + ) -> AbstractDataset: + try: + return AbstractDataset.from_config(name, config, load_version, save_version) + except DatasetError: + return UnavailableDataset() diff --git a/package/kedro_viz/integrations/kedro/data_loader.py b/package/kedro_viz/integrations/kedro/data_loader.py index 1ac1521e61..2955d73b29 100644 --- a/package/kedro_viz/integrations/kedro/data_loader.py +++ b/package/kedro_viz/integrations/kedro/data_loader.py @@ -7,8 +7,10 @@ import json import logging +import sys from pathlib import Path -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional, Set, Tuple +from unittest.mock import patch from kedro import __version__ from kedro.framework.project import configure_project, pipelines @@ -19,30 +21,14 @@ from kedro.pipeline import Pipeline from kedro_viz.constants import VIZ_METADATA_ARGS +from kedro_viz.integrations.kedro.abstract_dataset_lite import AbstractDatasetLite +from kedro_viz.integrations.kedro.lite_parser import LiteParser +from kedro_viz.integrations.utils import _VizNullPluginManager +from kedro_viz.models.metadata import Metadata logger = logging.getLogger(__name__) -class _VizNullPluginManager: - """This class creates an empty ``hook_manager`` that will ignore all calls to hooks - and registered plugins allowing the runner to function if no ``hook_manager`` - has been instantiated. - - NOTE: _VizNullPluginManager is a clone of _NullPluginManager class in Kedro. - This was introduced to support the earliest version of Kedro which does not - have _NullPluginManager defined - """ - - def __init__(self, *args, **kwargs): - pass - - def __getattr__(self, name): - return self - - def __call__(self, *args, **kwargs): - pass - - def _get_dataset_stats(project_path: Path) -> Dict: """Return the stats saved at stats.json as a dictionary if found. If not, return an empty dictionary @@ -69,33 +55,29 @@ def _get_dataset_stats(project_path: Path) -> Dict: return {} -def load_data( +def _load_data_helper( project_path: Path, env: Optional[str] = None, include_hooks: bool = False, - package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, -) -> Tuple[DataCatalog, Dict[str, Pipeline], BaseSessionStore, Dict]: - """Load data from a Kedro project. + is_lite: bool = False, +): + """Helper to load data from a Kedro project. + Args: project_path: the path where the Kedro project is located. env: the Kedro environment to load the data. If not provided. it will use Kedro default, which is local. include_hooks: A flag to include all registered hooks in your Kedro Project. - package_name: The name of the current package extra_params: Optional dictionary containing extra project parameters for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project configuration. + is_lite: A flag to run Kedro-Viz in lite mode. Returns: - A tuple containing the data catalog and the pipeline dictionary - and the session store. + A tuple containing the data catalog, pipeline dictionary, session store + and dataset stats dictionary. """ - if package_name: - configure_project(package_name) - else: - # bootstrap project when viz is run in dev mode - bootstrap_project(project_path) with KedroSession.create( project_path=project_path, @@ -109,12 +91,84 @@ def load_data( context = session.load_context() session_store = session._store - catalog = context.catalog + + # patch the AbstractDataset class for a custom + # implementation to handle kedro.io.core.DatasetError + if is_lite: + with patch("kedro.io.data_catalog.AbstractDataset", AbstractDatasetLite): + catalog = context.catalog + else: + catalog = context.catalog # Pipelines is a lazy dict-like object, so we force it to populate here # in case user doesn't have an active session down the line when it's first accessed. # Useful for users who have `get_current_session` in their `register_pipelines()`. pipelines_dict = dict(pipelines) stats_dict = _get_dataset_stats(project_path) - return catalog, pipelines_dict, session_store, stats_dict + + +def load_data( + project_path: Path, + env: Optional[str] = None, + include_hooks: bool = False, + package_name: Optional[str] = None, + extra_params: Optional[Dict[str, Any]] = None, + is_lite: bool = False, +) -> Tuple[DataCatalog, Dict[str, Pipeline], BaseSessionStore, Dict]: + """Load data from a Kedro project. + Args: + project_path: the path where the Kedro project is located. + env: the Kedro environment to load the data. If not provided. + it will use Kedro default, which is local. + include_hooks: A flag to include all registered hooks in your Kedro Project. + package_name: The name of the current package + extra_params: Optional dictionary containing extra project parameters + for underlying KedroContext. If specified, will update (and therefore + take precedence over) the parameters retrieved from the project + configuration. + is_lite: A flag to run Kedro-Viz in lite mode. + Returns: + A tuple containing the data catalog, pipeline dictionary, session store + and dataset stats dictionary. + """ + if package_name: + configure_project(package_name) + else: + # bootstrap project when viz is run in dev mode + bootstrap_project(project_path) + + if is_lite: + lite_parser = LiteParser(package_name) + unresolved_imports = lite_parser.parse(project_path) + sys_modules_patch = sys.modules.copy() + + if unresolved_imports and len(unresolved_imports) > 0: + modules_to_mock: Set[str] = set() + + # for the viz lite banner + Metadata.set_has_missing_dependencies(True) + + for unresolved_module_set in unresolved_imports.values(): + modules_to_mock = modules_to_mock.union(unresolved_module_set) + + mocked_modules = lite_parser.create_mock_modules(modules_to_mock) + sys_modules_patch.update(mocked_modules) + + logger.warning( + "Kedro-Viz is running with limited functionality. " + "For the best experience with full functionality, please\n" + "install the missing Kedro project dependencies:\n" + "%s \n", + list(mocked_modules.keys()), + ) + + # Patch actual sys modules + with patch.dict("sys.modules", sys_modules_patch): + return _load_data_helper( + project_path, env, include_hooks, extra_params, is_lite + ) + else: + return _load_data_helper( + project_path, env, include_hooks, extra_params, is_lite + ) diff --git a/package/kedro_viz/integrations/kedro/lite_parser.py b/package/kedro_viz/integrations/kedro/lite_parser.py new file mode 100755 index 0000000000..9fe619fe5c --- /dev/null +++ b/package/kedro_viz/integrations/kedro/lite_parser.py @@ -0,0 +1,273 @@ +"""`kedro_viz.integrations.kedro.lite_parser` defines a Kedro parser using AST.""" + +import ast +import importlib.util +import logging +from pathlib import Path +from typing import Dict, List, Set, Union +from unittest.mock import MagicMock + +logger = logging.getLogger(__name__) + + +class LiteParser: + """Represents a Kedro Parser which uses AST + + Args: + package_name (Union[str, None]): The name of the current package + """ + + def __init__(self, package_name: Union[str, None] = None) -> None: + self._package_name = package_name + + @staticmethod + def _is_module_importable(module_name: str) -> bool: + """Checks if a module is importable + + Args: + module_name (str): The name of the module to check + importability + Returns: + Whether the module can be imported + """ + try: + # Check if the module can be importable + # In case of submodule (contains a dot, e.g: sklearn.linear_model), + # find_spec imports the parent module + if importlib.util.find_spec(module_name) is None: + return False + return True + except ModuleNotFoundError as mnf_exc: + logger.debug( + "ModuleNotFoundError in resolving %s : %s", module_name, mnf_exc + ) + return False + except ImportError as imp_exc: + logger.debug("ImportError in resolving %s : %s", module_name, imp_exc) + return False + except ValueError as val_exc: + logger.debug("ValueError in resolving %s : %s", module_name, val_exc) + return False + # pylint: disable=broad-except + except Exception as exc: # pragma: no cover + logger.debug( + "An exception occurred while resolving %s : %s", module_name, exc + ) + return False + + @staticmethod + def _get_module_parts(module_name: str) -> List[str]: + """Creates a list of module parts to check for importability + + Args: + module_name (str): The module name to split + + Returns: + A list of module parts + + Example: + >>> LiteParser._get_module_parts("kedro.framework.project") + ["kedro", "kedro.framework", "kedro.framework.project"] + + """ + module_split = module_name.split(".") + full_module_name = "" + module_parts = [] + + for idx, sub_module_name in enumerate(module_split): + full_module_name = ( + sub_module_name if idx == 0 else f"{full_module_name}.{sub_module_name}" + ) + module_parts.append(full_module_name) + + return module_parts + + def _is_relative_import(self, module_name: str, project_file_paths: Set[Path]): + """Checks if a module is a relative import. This is needed + in dev or standalone mode when the package_name is None and + internal package files have unresolved external dependencies + + Args: + module_name (str): The name of the module to check + importability + project_file_paths (Set[Path]): A set of project file paths + + Returns: + Whether the module is a relative import starting + from the root package dir + + Example: + >>> lite_parser_obj = LiteParser() + >>> module_name = "kedro_project_package.pipelines.reporting.nodes" + >>> project_file_paths = set([Path("/path/to/relative/file")]) + >>> lite_parser_obj._is_relative_import(module_name, project_file_paths) + True + """ + relative_module_path = str(Path(*module_name.split("."))) + + # Check if the relative_module_path + # is a substring of current project file path + is_relative_import_path = any( + relative_module_path in str(project_file_path) + for project_file_path in project_file_paths + ) + + return is_relative_import_path + + def _populate_missing_dependencies( + self, module_name: str, missing_dependencies: Set[str] + ) -> None: + """Helper to populate missing dependencies + + Args: + module_name (str): The module name to check if it is importable + missing_dependencies (Set[str]): A set of missing dependencies + + """ + module_name_parts = self._get_module_parts(module_name) + for module_name_part in module_name_parts: + if ( + not self._is_module_importable(module_name_part) + and module_name_part not in missing_dependencies + ): + missing_dependencies.add(module_name_part) + + def _get_unresolved_imports( + self, file_path: Path, project_file_paths: Union[Set[Path], None] = None + ) -> Set[str]: + """Parse the file using AST and return any missing dependencies + in the current file + + Args: + file_path (Path): The file path to parse + project_file_paths Union[Set[Path], None]: A set of project file paths + + Returns: + A set of missing dependencies + """ + + missing_dependencies: Set[str] = set() + + # Read the file + with open(file_path, "r", encoding="utf-8") as file: + file_content = file.read() + + # parse file content using ast + parsed_content_ast_node: ast.Module = ast.parse(file_content) + file_path = file_path.resolve() + + # Explore each node in the AST tree + for node in ast.walk(parsed_content_ast_node): + # Handling dependencies that starts with "import " + # Example: import logging + # Corresponding AST node will be: + # Import(names=[alias(name='logging')]) + if isinstance(node, ast.Import): + for alias in node.names: + module_name = alias.name + self._populate_missing_dependencies( + module_name, missing_dependencies + ) + + # Handling dependencies that starts with "from " + # Example: from typing import Dict, Union + # Corresponding AST node will be: + # ImportFrom(module='typing', names=[alias(name='Dict'), + # alias(name='Union')], + # level=0) + elif isinstance(node, ast.ImportFrom): + module_name = node.module if node.module else "" + level = node.level + + # Ignore relative imports like "from . import a" + if not module_name: + continue + + # Ignore relative imports within the package + # Examples: + # "from demo_project.pipelines.reporting import test", + # "from ..nodes import func_test" + if (self._package_name and self._package_name in module_name) or ( + # dev or standalone mode + not self._package_name + and project_file_paths + and self._is_relative_import(module_name, project_file_paths) + ): + continue + + # absolute modules in the env + # Examples: + # from typing import Dict, Union + # from sklearn.linear_model import LinearRegression + if level == 0: + self._populate_missing_dependencies( + module_name, missing_dependencies + ) + + return missing_dependencies + + def create_mock_modules(self, unresolved_imports: Set[str]) -> Dict[str, MagicMock]: + """Creates mock modules for unresolved imports + + Args: + unresolved_imports (Set[str]): A set of unresolved imports + + Returns: + A dictionary of mocked modules for the unresolved imports + """ + mocked_modules: Dict[str, MagicMock] = {} + + for unresolved_import in unresolved_imports: + mocked_modules[unresolved_import] = MagicMock() + + return mocked_modules + + def parse(self, target_path: Path) -> Union[Dict[str, Set[str]], None]: + """Parses the file(s) in the target path and returns + any unresolved imports for all the dependency errors + as a dictionary of file(s) in the target path and a set of module names + + Args: + target_path (Path): The path to parse file(s) + + Returns: + A dictionary of file(s) in the target path and a set of module names + """ + + if not target_path.exists(): + logger.warning("Path `%s` does not exist", str(target_path)) + return None + + unresolved_imports: Dict[str, Set[str]] = {} + + if target_path.is_file(): + missing_dependencies = self._get_unresolved_imports(target_path) + if len(missing_dependencies) > 0: + unresolved_imports[str(target_path)] = missing_dependencies + return unresolved_imports + + # handling directories + _project_file_paths = set(target_path.rglob("*.py")) + + for file_path in _project_file_paths: + try: + # Ensure the package name is in the file path + if self._package_name and self._package_name not in file_path.parts: + # we are only mocking the dependencies + # inside the package + continue + + missing_dependencies = self._get_unresolved_imports( + file_path, _project_file_paths + ) + if len(missing_dependencies) > 0: + unresolved_imports[str(file_path)] = missing_dependencies + # pylint: disable=broad-except + except Exception as exc: # pragma: no cover + logger.error( + "An error occurred in LiteParser while mocking dependencies : %s", + exc, + ) + continue + + return unresolved_imports diff --git a/package/kedro_viz/integrations/utils.py b/package/kedro_viz/integrations/utils.py new file mode 100644 index 0000000000..1875cd7a85 --- /dev/null +++ b/package/kedro_viz/integrations/utils.py @@ -0,0 +1,57 @@ +"""`kedro_viz.integrations.utils` provides utility functions and classes +to integrate Kedro-Viz with external data sources. +""" + +from typing import Any, Union + +from kedro.io.core import AbstractDataset + +_EMPTY = object() + + +class _VizNullPluginManager: # pragma: no cover + """This class creates an empty ``hook_manager`` that will ignore all calls to hooks + and registered plugins allowing the runner to function if no ``hook_manager`` + has been instantiated. + + NOTE: _VizNullPluginManager is a clone of _NullPluginManager class in Kedro. + This was introduced to support the earliest version of Kedro which does not + have _NullPluginManager defined + """ + + def __init__(self, *args, **kwargs): + pass + + def __getattr__(self, name): + return self + + def __call__(self, *args, **kwargs): + pass + + +class UnavailableDataset(AbstractDataset): # pragma: no cover + """This class is a custom dataset implementation for `Kedro Viz Lite` + when kedro-datasets are unavailable""" + + def __init__( + self, + data: Any = _EMPTY, + metadata: Union[dict[str, Any], None] = None, + ): + self._data = data + self.metadata = metadata + + def _load(self, *args, **kwargs): + pass + + def _save(self, *args, **kwargs): + pass + + load = _load + save = _save + + def _exists(self): + pass + + def _describe(self) -> dict[str, Any]: + return {"data": self._data} diff --git a/package/kedro_viz/launchers/cli.py b/package/kedro_viz/launchers/cli.py deleted file mode 100644 index 5eac79907d..0000000000 --- a/package/kedro_viz/launchers/cli.py +++ /dev/null @@ -1,418 +0,0 @@ -"""`kedro_viz.launchers.cli` launches the viz server as a CLI app.""" - -import multiprocessing -import traceback -from pathlib import Path -from typing import Dict - -import click -from click_default_group import DefaultGroup -from kedro.framework.cli.project import PARAMS_ARG_HELP -from kedro.framework.cli.utils import KedroCliError, _split_params -from kedro.framework.project import PACKAGE_NAME -from packaging.version import parse -from watchgod import RegExpWatcher, run_process - -from kedro_viz import __version__ -from kedro_viz.constants import ( - DEFAULT_HOST, - DEFAULT_PORT, - SHAREABLEVIZ_SUPPORTED_PLATFORMS, - VIZ_DEPLOY_TIME_LIMIT, -) -from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory -from kedro_viz.integrations.pypi import get_latest_version, is_running_outdated_version -from kedro_viz.launchers.utils import ( - _PYPROJECT, - _check_viz_up, - _find_kedro_project, - _start_browser, - _wait_for, - viz_deploy_progress_timer, -) -from kedro_viz.server import load_and_populate_data - -try: - from azure.core.exceptions import ServiceRequestError -except ImportError: # pragma: no cover - ServiceRequestError = None # type: ignore - -_VIZ_PROCESSES: Dict[str, int] = {} - - -@click.group(name="Kedro-Viz") -def viz_cli(): # pylint: disable=missing-function-docstring - pass - - -@viz_cli.group(cls=DefaultGroup, default="run", default_if_no_args=True) -@click.pass_context -def viz(ctx): # pylint: disable=unused-argument - """Visualise a Kedro pipeline using Kedro viz.""" - - -@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--host", - default=DEFAULT_HOST, - help="Host that viz will listen to. Defaults to localhost.", -) -@click.option( - "--port", - default=DEFAULT_PORT, - type=int, - help="TCP port that viz will listen to. Defaults to 4141.", -) -@click.option( - "--browser/--no-browser", - default=True, - help="Whether to open viz interface in the default browser or not. " - "Browser will only be opened if host is localhost. Defaults to True.", -) -@click.option( - "--load-file", - default=None, - help="Path to load Kedro-Viz data from a directory", -) -@click.option( - "--save-file", - default=None, - type=click.Path(dir_okay=False, writable=True), - help="Path to save Kedro-Viz data to a directory", -) -@click.option( - "--pipeline", - "-p", - type=str, - default=None, - help="Name of the registered pipeline to visualise. " - "If not set, the default pipeline is visualised", -) -@click.option( - "--env", - "-e", - type=str, - default=None, - multiple=False, - envvar="KEDRO_ENV", - help="Kedro configuration environment. If not specified, " - "catalog config in `local` will be used", -) -@click.option( - "--autoreload", - "-a", - is_flag=True, - help="Autoreload viz server when a Python or YAML file change in the Kedro project", -) -@click.option( - "--include-hooks", - is_flag=True, - help="A flag to include all registered hooks in your Kedro Project", -) -@click.option( - "--params", - type=click.UNPROCESSED, - default="", - help=PARAMS_ARG_HELP, - callback=_split_params, -) -# pylint: disable=import-outside-toplevel, too-many-locals -def run( - host, - port, - browser, - load_file, - save_file, - pipeline, - env, - autoreload, - include_hooks, - params, -): - """Launch local Kedro Viz instance""" - from kedro_viz.server import run_server - - kedro_project_path = _find_kedro_project(Path.cwd()) - - if kedro_project_path is None: - display_cli_message( - "ERROR: Failed to start Kedro-Viz : " - "Could not find the project configuration " - f"file '{_PYPROJECT}' at '{Path.cwd()}'. ", - "red", - ) - return - - installed_version = parse(__version__) - latest_version = get_latest_version() - if is_running_outdated_version(installed_version, latest_version): - display_cli_message( - "WARNING: You are using an old version of Kedro Viz. " - f"You are using version {installed_version}; " - f"however, version {latest_version} is now available.\n" - "You should consider upgrading via the `pip install -U kedro-viz` command.\n" - "You can view the complete changelog at " - "https://github.com/kedro-org/kedro-viz/releases.", - "yellow", - ) - try: - if port in _VIZ_PROCESSES and _VIZ_PROCESSES[port].is_alive(): - _VIZ_PROCESSES[port].terminate() - - run_server_kwargs = { - "host": host, - "port": port, - "load_file": load_file, - "save_file": save_file, - "pipeline_name": pipeline, - "env": env, - "project_path": kedro_project_path, - "autoreload": autoreload, - "include_hooks": include_hooks, - "package_name": PACKAGE_NAME, - "extra_params": params, - } - if autoreload: - run_process_kwargs = { - "path": kedro_project_path, - "target": run_server, - "kwargs": run_server_kwargs, - "watcher_cls": RegExpWatcher, - "watcher_kwargs": {"re_files": r"^.*(\.yml|\.yaml|\.py|\.json)$"}, - } - viz_process = multiprocessing.Process( - target=run_process, daemon=False, kwargs={**run_process_kwargs} - ) - else: - viz_process = multiprocessing.Process( - target=run_server, daemon=False, kwargs={**run_server_kwargs} - ) - - display_cli_message("Starting Kedro Viz ...", "green") - - viz_process.start() - - _VIZ_PROCESSES[port] = viz_process - - _wait_for(func=_check_viz_up, host=host, port=port) - - display_cli_message( - "Kedro Viz started successfully. \n\n" - f"\u2728 Kedro Viz is running at \n http://{host}:{port}/", - "green", - ) - - if browser: - _start_browser(host, port) - - except Exception as ex: # pragma: no cover - traceback.print_exc() - raise KedroCliError(str(ex)) from ex - - -@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--platform", - type=str, - required=True, - help=f"Supported Cloud Platforms like {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,} to host Kedro Viz", -) -@click.option( - "--endpoint", - type=str, - required=True, - help="Static Website hosted endpoint." - "(eg., For AWS - http://.s3-website..amazonaws.com/)", -) -@click.option( - "--bucket-name", - type=str, - required=True, - help="Bucket name where Kedro Viz will be hosted", -) -@click.option( - "--include-hooks", - is_flag=True, - help="A flag to include all registered hooks in your Kedro Project", -) -@click.option( - "--include-previews", - is_flag=True, - help="A flag to include preview for all the datasets", -) -def deploy(platform, endpoint, bucket_name, include_hooks, include_previews): - """Deploy and host Kedro Viz on provided platform""" - if not platform or platform.lower() not in SHAREABLEVIZ_SUPPORTED_PLATFORMS: - display_cli_message( - "ERROR: Invalid platform specified. Kedro-Viz supports \n" - f"the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}", - "red", - ) - return - - if not endpoint: - display_cli_message( - "ERROR: Invalid endpoint specified. If you are looking for platform \n" - "agnostic shareable viz solution, please use the `kedro viz build` command", - "red", - ) - return - - create_shareableviz_process( - platform, - include_previews, - endpoint, - bucket_name, - include_hooks, - ) - - -@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) -@click.option( - "--include-hooks", - is_flag=True, - help="A flag to include all registered hooks in your Kedro Project", -) -@click.option( - "--include-previews", - is_flag=True, - help="A flag to include preview for all the datasets", -) -def build(include_hooks, include_previews): - """Create build directory of local Kedro Viz instance with Kedro project data""" - - create_shareableviz_process("local", include_previews, include_hooks=include_hooks) - - -def create_shareableviz_process( - platform, - is_all_previews_enabled, - endpoint=None, - bucket_name=None, - include_hooks=False, -): - """Creates platform specific deployer process""" - try: - process_completed = multiprocessing.Value("i", 0) - exception_queue = multiprocessing.Queue() - - viz_deploy_process = multiprocessing.Process( - target=load_and_deploy_viz, - args=( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - PACKAGE_NAME, - process_completed, - exception_queue, - ), - ) - - viz_deploy_process.start() - viz_deploy_progress_timer(process_completed, VIZ_DEPLOY_TIME_LIMIT) - - if not exception_queue.empty(): # pragma: no cover - raise exception_queue.get_nowait() - - if not process_completed.value: - raise TimeoutError() - - if platform != "local": - display_cli_message( - f"\u2728 Success! Kedro Viz has been deployed on {platform.upper()}. " - "It can be accessed at :\n" - f"{endpoint}", - "green", - ) - else: - display_cli_message( - "\u2728 Success! Kedro-Viz build files have been " - "added to the `build` directory.", - "green", - ) - - except TimeoutError: # pragma: no cover - display_cli_message( - "TIMEOUT ERROR: Failed to build/deploy Kedro-Viz as the " - f"process took more than {VIZ_DEPLOY_TIME_LIMIT} seconds. " - "Please try again later.", - "red", - ) - - except KeyboardInterrupt: # pragma: no cover - display_cli_message( - "\nCreating your build/deploy Kedro-Viz process " - "is interrupted. Exiting...", - "red", - ) - - except PermissionError: # pragma: no cover - if platform != "local": - display_cli_message( - "PERMISSION ERROR: Deploying and hosting Kedro-Viz requires " - f"{platform.upper()} access keys, a valid {platform.upper()} " - "endpoint and bucket name.\n" - f"Please supply your {platform.upper()} access keys as environment variables " - f"and make sure the {platform.upper()} endpoint and bucket name are valid.\n" - "More information can be found at : " - "https://docs.kedro.org/en/stable/visualisation/share_kedro_viz.html", - "red", - ) - else: - display_cli_message( - "PERMISSION ERROR: Please make sure, " - "you have write access to the current directory", - "red", - ) - # pylint: disable=broad-exception-caught - except Exception as exc: # pragma: no cover - display_cli_message(f"ERROR: Failed to build/deploy Kedro-Viz : {exc} ", "red") - - finally: - viz_deploy_process.terminate() - - -def load_and_deploy_viz( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - package_name, - process_completed, - exception_queue, -): - """Loads Kedro Project data, creates a deployer and deploys to a platform""" - try: - load_and_populate_data( - Path.cwd(), include_hooks=include_hooks, package_name=package_name - ) - - # Start the deployment - deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) - deployer.deploy(is_all_previews_enabled) - - except ( - # pylint: disable=catching-non-exception - (FileNotFoundError, ServiceRequestError) - if ServiceRequestError is not None - else FileNotFoundError - ): # pragma: no cover - exception_queue.put(Exception("The specified bucket does not exist")) - # pylint: disable=broad-exception-caught - except Exception as exc: # pragma: no cover - exception_queue.put(exc) - finally: - process_completed.value = 1 - - -def display_cli_message(msg, msg_color=None): - """Displays message for Kedro Viz build and deploy commands""" - click.echo( - click.style( - msg, - fg=msg_color, - ) - ) diff --git a/package/kedro_viz/launchers/cli/__init__.py b/package/kedro_viz/launchers/cli/__init__.py new file mode 100644 index 0000000000..56806ffefd --- /dev/null +++ b/package/kedro_viz/launchers/cli/__init__.py @@ -0,0 +1 @@ +"""`kedro_viz.launchers.cli` launches the viz server as a CLI app.""" diff --git a/package/kedro_viz/launchers/cli/build.py b/package/kedro_viz/launchers/cli/build.py new file mode 100644 index 0000000000..d506266019 --- /dev/null +++ b/package/kedro_viz/launchers/cli/build.py @@ -0,0 +1,24 @@ +"""`kedro_viz.launchers.cli.build` provides a cli command to build +a Kedro-Viz instance""" +# pylint: disable=import-outside-toplevel +import click + +from kedro_viz.launchers.cli.main import viz + + +@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) +@click.option( + "--include-hooks", + is_flag=True, + help="A flag to include all registered hooks in your Kedro Project", +) +@click.option( + "--include-previews", + is_flag=True, + help="A flag to include preview for all the datasets", +) +def build(include_hooks, include_previews): + """Create build directory of local Kedro Viz instance with Kedro project data""" + from kedro_viz.launchers.cli.utils import create_shareableviz_process + + create_shareableviz_process("local", include_previews, include_hooks=include_hooks) diff --git a/package/kedro_viz/launchers/cli/deploy.py b/package/kedro_viz/launchers/cli/deploy.py new file mode 100644 index 0000000000..10bb31870f --- /dev/null +++ b/package/kedro_viz/launchers/cli/deploy.py @@ -0,0 +1,69 @@ +"""`kedro_viz.launchers.cli.deploy` provides a cli command to deploy +a Kedro-Viz instance on cloud platforms""" +# pylint: disable=import-outside-toplevel +import click + +from kedro_viz.constants import SHAREABLEVIZ_SUPPORTED_PLATFORMS +from kedro_viz.launchers.cli.main import viz + + +@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) +@click.option( + "--platform", + type=str, + required=True, + help=f"Supported Cloud Platforms like {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,} to host Kedro Viz", +) +@click.option( + "--endpoint", + type=str, + required=True, + help="Static Website hosted endpoint." + "(eg., For AWS - http://.s3-website..amazonaws.com/)", +) +@click.option( + "--bucket-name", + type=str, + required=True, + help="Bucket name where Kedro Viz will be hosted", +) +@click.option( + "--include-hooks", + is_flag=True, + help="A flag to include all registered hooks in your Kedro Project", +) +@click.option( + "--include-previews", + is_flag=True, + help="A flag to include preview for all the datasets", +) +def deploy(platform, endpoint, bucket_name, include_hooks, include_previews): + """Deploy and host Kedro Viz on provided platform""" + from kedro_viz.launchers.cli.utils import ( + create_shareableviz_process, + display_cli_message, + ) + + if not platform or platform.lower() not in SHAREABLEVIZ_SUPPORTED_PLATFORMS: + display_cli_message( + "ERROR: Invalid platform specified. Kedro-Viz supports \n" + f"the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}", + "red", + ) + return + + if not endpoint: + display_cli_message( + "ERROR: Invalid endpoint specified. If you are looking for platform \n" + "agnostic shareable viz solution, please use the `kedro viz build` command", + "red", + ) + return + + create_shareableviz_process( + platform, + include_previews, + endpoint, + bucket_name, + include_hooks, + ) diff --git a/package/kedro_viz/launchers/cli/lazy_default_group.py b/package/kedro_viz/launchers/cli/lazy_default_group.py new file mode 100644 index 0000000000..861d023221 --- /dev/null +++ b/package/kedro_viz/launchers/cli/lazy_default_group.py @@ -0,0 +1,76 @@ +"""`kedro_viz.launchers.cli.lazy_default_group` provides a custom mutli-command +subclass for a lazy subcommand loader""" + +# pylint: disable=import-outside-toplevel +from typing import Any, Union + +import click + + +class LazyDefaultGroup(click.Group): + """A click Group that supports lazy loading of subcommands and a default command""" + + def __init__( + self, + *args: Any, + **kwargs: Any, + ): + if not kwargs.get("ignore_unknown_options", True): + raise ValueError("Default group accepts unknown options") + self.ignore_unknown_options = True + + # lazy_subcommands is a map of the form: + # + # {command-name} -> {module-name}.{command-object-name} + # + self.lazy_subcommands = kwargs.pop("lazy_subcommands", {}) + + self.default_cmd_name = kwargs.pop("default", None) + self.default_if_no_args = kwargs.pop("default_if_no_args", False) + + super().__init__(*args, **kwargs) + + def list_commands(self, ctx: click.Context) -> list[str]: + return sorted(self.lazy_subcommands.keys()) + + def get_command( # type: ignore[override] + self, ctx: click.Context, cmd_name: str + ) -> Union[click.BaseCommand, click.Command, None]: + if cmd_name in self.lazy_subcommands: + return self._lazy_load(cmd_name) + return super().get_command(ctx, cmd_name) + + def _lazy_load(self, cmd_name: str) -> click.BaseCommand: + from importlib import import_module + + # lazily loading a command, first get the module name and attribute name + import_path = self.lazy_subcommands[cmd_name] + modname, cmd_object_name = import_path.rsplit(".", 1) + + # do the import + mod = import_module(modname) + + # get the Command object from that module + cmd_object = getattr(mod, cmd_object_name) + + return cmd_object + + def parse_args(self, ctx, args): + # If no args are provided and default_command_name is specified, + # use the default command + if not args and self.default_if_no_args: + args.insert(0, self.default_cmd_name) + return super().parse_args(ctx, args) + + def resolve_command(self, ctx: click.Context, args): + # Attempt to resolve the command using the parent class method + try: + cmd_name, cmd, args = super().resolve_command(ctx, args) + return cmd_name, cmd, args + except click.UsageError as exc: + if self.default_cmd_name and not ctx.invoked_subcommand: + # No command found, use the default command + default_cmd = self.get_command(ctx, self.default_cmd_name) + if default_cmd: + return default_cmd.name, default_cmd, args + raise exc diff --git a/package/kedro_viz/launchers/cli/main.py b/package/kedro_viz/launchers/cli/main.py new file mode 100644 index 0000000000..0ccb1515e1 --- /dev/null +++ b/package/kedro_viz/launchers/cli/main.py @@ -0,0 +1,26 @@ +"""`kedro_viz.launchers.cli.main` is an entry point for Kedro-Viz cli commands.""" + +import click + +from kedro_viz.launchers.cli.lazy_default_group import LazyDefaultGroup + + +@click.group(name="Kedro-Viz") +def viz_cli(): # pylint: disable=missing-function-docstring + pass + + +@viz_cli.group( + name="viz", + cls=LazyDefaultGroup, + lazy_subcommands={ + "run": "kedro_viz.launchers.cli.run.run", + "deploy": "kedro_viz.launchers.cli.deploy.deploy", + "build": "kedro_viz.launchers.cli.build.build", + }, + default="run", + default_if_no_args=True, +) +@click.pass_context +def viz(ctx): # pylint: disable=unused-argument + """Visualise a Kedro pipeline using Kedro viz.""" diff --git a/package/kedro_viz/launchers/cli/run.py b/package/kedro_viz/launchers/cli/run.py new file mode 100644 index 0000000000..9988214261 --- /dev/null +++ b/package/kedro_viz/launchers/cli/run.py @@ -0,0 +1,203 @@ +"""`kedro_viz.launchers.cli.run` provides a cli command to run +a Kedro-Viz instance""" + +from typing import Dict + +import click +from kedro.framework.cli.project import PARAMS_ARG_HELP +from kedro.framework.cli.utils import _split_params + +from kedro_viz.constants import DEFAULT_HOST, DEFAULT_PORT +from kedro_viz.launchers.cli.main import viz + +_VIZ_PROCESSES: Dict[str, int] = {} + + +@viz.command(context_settings={"help_option_names": ["-h", "--help"]}) +@click.option( + "--host", + default=DEFAULT_HOST, + help="Host that viz will listen to. Defaults to localhost.", +) +@click.option( + "--port", + default=DEFAULT_PORT, + type=int, + help="TCP port that viz will listen to. Defaults to 4141.", +) +@click.option( + "--browser/--no-browser", + default=True, + help="Whether to open viz interface in the default browser or not. " + "Browser will only be opened if host is localhost. Defaults to True.", +) +@click.option( + "--load-file", + default=None, + help="Path to load Kedro-Viz data from a directory", +) +@click.option( + "--save-file", + default=None, + type=click.Path(dir_okay=False, writable=True), + help="Path to save Kedro-Viz data to a directory", +) +@click.option( + "--pipeline", + "-p", + type=str, + default=None, + help="Name of the registered pipeline to visualise. " + "If not set, the default pipeline is visualised", +) +@click.option( + "--env", + "-e", + type=str, + default=None, + multiple=False, + envvar="KEDRO_ENV", + help="Kedro configuration environment. If not specified, " + "catalog config in `local` will be used", +) +@click.option( + "--autoreload", + "-a", + is_flag=True, + help="Autoreload viz server when a Python or YAML file change in the Kedro project", +) +@click.option( + "--include-hooks", + is_flag=True, + help="A flag to include all registered hooks in your Kedro Project", +) +@click.option( + "--params", + type=click.UNPROCESSED, + default="", + help=PARAMS_ARG_HELP, + callback=_split_params, +) +@click.option( + "--lite", + is_flag=True, + help="An experimental flag to open Kedro-Viz without Kedro project dependencies", +) +# pylint: disable=import-outside-toplevel, too-many-locals +def run( + host, + port, + browser, + load_file, + save_file, + pipeline, + env, + autoreload, + include_hooks, + params, + lite, +): + """Launch local Kedro Viz instance""" + # Deferring Imports + import multiprocessing + import traceback + from pathlib import Path + + from kedro.framework.cli.utils import KedroCliError + from kedro.framework.project import PACKAGE_NAME + from packaging.version import parse + + from kedro_viz import __version__ + from kedro_viz.integrations.pypi import ( + get_latest_version, + is_running_outdated_version, + ) + from kedro_viz.launchers.cli.utils import display_cli_message + from kedro_viz.launchers.utils import ( + _PYPROJECT, + _check_viz_up, + _find_kedro_project, + _start_browser, + _wait_for, + ) + from kedro_viz.server import run_server + + kedro_project_path = _find_kedro_project(Path.cwd()) + + if kedro_project_path is None: + display_cli_message( + "ERROR: Failed to start Kedro-Viz : " + "Could not find the project configuration " + f"file '{_PYPROJECT}' at '{Path.cwd()}'. ", + "red", + ) + return + + installed_version = parse(__version__) + latest_version = get_latest_version() + if is_running_outdated_version(installed_version, latest_version): + display_cli_message( + "WARNING: You are using an old version of Kedro Viz. " + f"You are using version {installed_version}; " + f"however, version {latest_version} is now available.\n" + "You should consider upgrading via the `pip install -U kedro-viz` command.\n" + "You can view the complete changelog at " + "https://github.com/kedro-org/kedro-viz/releases.", + "yellow", + ) + try: + if port in _VIZ_PROCESSES and _VIZ_PROCESSES[port].is_alive(): + _VIZ_PROCESSES[port].terminate() + + run_server_kwargs = { + "host": host, + "port": port, + "load_file": load_file, + "save_file": save_file, + "pipeline_name": pipeline, + "env": env, + "project_path": kedro_project_path, + "autoreload": autoreload, + "include_hooks": include_hooks, + "package_name": PACKAGE_NAME, + "extra_params": params, + "is_lite": lite, + } + if autoreload: + from watchgod import RegExpWatcher, run_process + + run_process_kwargs = { + "path": kedro_project_path, + "target": run_server, + "kwargs": run_server_kwargs, + "watcher_cls": RegExpWatcher, + "watcher_kwargs": {"re_files": r"^.*(\.yml|\.yaml|\.py|\.json)$"}, + } + viz_process = multiprocessing.Process( + target=run_process, daemon=False, kwargs={**run_process_kwargs} + ) + else: + viz_process = multiprocessing.Process( + target=run_server, daemon=False, kwargs={**run_server_kwargs} + ) + + display_cli_message("Starting Kedro Viz ...", "green") + + viz_process.start() + + _VIZ_PROCESSES[port] = viz_process + + _wait_for(func=_check_viz_up, host=host, port=port) + + display_cli_message( + "Kedro Viz started successfully. \n\n" + f"\u2728 Kedro Viz is running at \n http://{host}:{port}/", + "green", + ) + + if browser: + _start_browser(host, port) + + except Exception as ex: # pragma: no cover + traceback.print_exc() + raise KedroCliError(str(ex)) from ex diff --git a/package/kedro_viz/launchers/cli/utils.py b/package/kedro_viz/launchers/cli/utils.py new file mode 100644 index 0000000000..eb4efdfbc9 --- /dev/null +++ b/package/kedro_viz/launchers/cli/utils.py @@ -0,0 +1,169 @@ +"""`kedro_viz.launchers.cli.utils` provides utility functions for cli commands.""" +# pylint: disable=import-outside-toplevel +from pathlib import Path +from time import sleep +from typing import Union + +import click + +from kedro_viz.constants import VIZ_DEPLOY_TIME_LIMIT + + +def create_shareableviz_process( + platform: str, + is_all_previews_enabled: bool, + endpoint: Union[str, None] = None, + bucket_name: Union[str, None] = None, + include_hooks: bool = False, +): + """Creates platform specific deployer process""" + + import multiprocessing + + from kedro.framework.project import PACKAGE_NAME + + try: + process_completed = multiprocessing.Value("i", 0) + exception_queue = multiprocessing.Queue() # type: ignore[var-annotated] + + viz_deploy_process = multiprocessing.Process( + target=_load_and_deploy_viz, + args=( + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + PACKAGE_NAME, + process_completed, + exception_queue, + ), + ) + + viz_deploy_process.start() + _viz_deploy_progress_timer(process_completed, VIZ_DEPLOY_TIME_LIMIT) + + if not exception_queue.empty(): # pragma: no cover + raise exception_queue.get_nowait() + + if not process_completed.value: + raise TimeoutError() + + if platform != "local": + display_cli_message( + f"\u2728 Success! Kedro Viz has been deployed on {platform.upper()}. " + "It can be accessed at :\n" + f"{endpoint}", + "green", + ) + else: + display_cli_message( + "\u2728 Success! Kedro-Viz build files have been " + "added to the `build` directory.", + "green", + ) + + except TimeoutError: # pragma: no cover + display_cli_message( + "TIMEOUT ERROR: Failed to build/deploy Kedro-Viz as the " + f"process took more than {VIZ_DEPLOY_TIME_LIMIT} seconds. " + "Please try again later.", + "red", + ) + + except KeyboardInterrupt: # pragma: no cover + display_cli_message( + "\nCreating your build/deploy Kedro-Viz process " + "is interrupted. Exiting...", + "red", + ) + + except PermissionError: # pragma: no cover + if platform != "local": + display_cli_message( + "PERMISSION ERROR: Deploying and hosting Kedro-Viz requires " + f"{platform.upper()} access keys, a valid {platform.upper()} " + "endpoint and bucket name.\n" + f"Please supply your {platform.upper()} access keys as environment variables " + f"and make sure the {platform.upper()} endpoint and bucket name are valid.\n" + "More information can be found at : " + "https://docs.kedro.org/en/stable/visualisation/share_kedro_viz.html", + "red", + ) + else: + display_cli_message( + "PERMISSION ERROR: Please make sure, " + "you have write access to the current directory", + "red", + ) + # pylint: disable=broad-exception-caught + except Exception as exc: # pragma: no cover + display_cli_message(f"ERROR: Failed to build/deploy Kedro-Viz : {exc} ", "red") + + finally: + viz_deploy_process.terminate() + + +def display_cli_message(msg, msg_color=None): + """Displays message for Kedro Viz build and deploy commands""" + click.echo( + click.style( + msg, + fg=msg_color, + ) + ) + + +def _load_and_deploy_viz( + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + package_name, + process_completed, + exception_queue, +): + """Loads Kedro Project data, creates a deployer and deploys to a platform""" + try: + from kedro_viz.integrations.deployment.deployer_factory import DeployerFactory + from kedro_viz.server import load_and_populate_data + + try: + from azure.core.exceptions import ServiceRequestError + except ImportError: # pragma: no cover + ServiceRequestError = None + + load_and_populate_data( + Path.cwd(), include_hooks=include_hooks, package_name=package_name + ) + + # Start the deployment + deployer = DeployerFactory.create_deployer(platform, endpoint, bucket_name) + deployer.deploy(is_all_previews_enabled) + + except ( + # pylint: disable=catching-non-exception + (FileNotFoundError, ServiceRequestError) + if ServiceRequestError is not None + else FileNotFoundError + ): # pragma: no cover + exception_queue.put(Exception("The specified bucket does not exist")) + # pylint: disable=broad-exception-caught + except Exception as exc: # pragma: no cover + exception_queue.put(exc) + finally: + process_completed.value = 1 + + +def _viz_deploy_progress_timer(process_completed, timeout): + """Shows progress timer and message for kedro viz deploy""" + elapsed_time = 0 + while elapsed_time <= timeout and not process_completed.value: + print( + f"...Creating your build/deploy Kedro-Viz ({elapsed_time}s)", + end="\r", + flush=True, + ) + sleep(1) + elapsed_time += 1 diff --git a/package/kedro_viz/launchers/utils.py b/package/kedro_viz/launchers/utils.py index bf79602349..c4b0076677 100755 --- a/package/kedro_viz/launchers/utils.py +++ b/package/kedro_viz/launchers/utils.py @@ -96,19 +96,6 @@ def _start_browser(host: str, port: int): webbrowser.open_new(f"http://{host}:{port}/") -def viz_deploy_progress_timer(process_completed, timeout): - """Shows progress timer and message for kedro viz deploy""" - elapsed_time = 0 - while elapsed_time <= timeout and not process_completed.value: - print( - f"...Creating your build/deploy Kedro-Viz ({elapsed_time}s)", - end="\r", - flush=True, - ) - sleep(1) - elapsed_time += 1 - - def _is_project(project_path: Union[str, Path]) -> bool: metadata_file = Path(project_path).expanduser().resolve() / _PYPROJECT if not metadata_file.is_file(): diff --git a/package/kedro_viz/models/flowchart.py b/package/kedro_viz/models/flowchart.py index ed55fcfa61..e8e81cfb61 100644 --- a/package/kedro_viz/models/flowchart.py +++ b/package/kedro_viz/models/flowchart.py @@ -452,14 +452,7 @@ def set_parameters(cls, _): @field_validator("run_command") @classmethod def set_run_command(cls, _): - # if a node doesn't have a user-supplied `_name` attribute, - # a human-readable run command `kedro run --to-nodes/nodes` is not available - if cls.kedro_node._name is not None: - if cls.task_node.namespace is not None: - return f"kedro run --to-nodes={cls.task_node.namespace}.{cls.kedro_node._name}" - return f"kedro run --to-nodes={cls.kedro_node._name}" - - return None + return f"kedro run --to-nodes='{cls.kedro_node.name}'" @field_validator("inputs") @classmethod diff --git a/package/kedro_viz/models/metadata.py b/package/kedro_viz/models/metadata.py new file mode 100644 index 0000000000..debe1f04e3 --- /dev/null +++ b/package/kedro_viz/models/metadata.py @@ -0,0 +1,47 @@ +"""`kedro_viz.models.metadata` defines metadata for Kedro-Viz application.""" + +# pylint: disable=missing-function-docstring +from typing import ClassVar, List + +from pydantic import BaseModel, field_validator + + +class PackageCompatibility(BaseModel): + """Represent package compatibility in app metadata""" + + package_name: str + package_version: str + is_compatible: bool + + @field_validator("package_name") + @classmethod + def set_package_name(cls, value): + assert isinstance(value, str) + return value + + @field_validator("package_version") + @classmethod + def set_package_version(cls, value): + assert isinstance(value, str) + return value + + @field_validator("is_compatible") + @classmethod + def set_is_compatible(cls, value): + assert isinstance(value, bool) + return value + + +class Metadata(BaseModel): + """Represent Kedro-Viz application metadata""" + + has_missing_dependencies: ClassVar[bool] = False + package_compatibilities: ClassVar[List[PackageCompatibility]] = [] + + @classmethod + def set_package_compatibilities(cls, value: List[PackageCompatibility]): + cls.package_compatibilities = value + + @classmethod + def set_has_missing_dependencies(cls, value: bool): + cls.has_missing_dependencies = value diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 384a3545dc..76026ddbbf 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -1,18 +1,13 @@ """`kedro_viz.server` provides utilities to launch a webserver for Kedro pipeline visualisation.""" -import multiprocessing from pathlib import Path from typing import Any, Dict, Optional -import fsspec -import uvicorn from kedro.framework.session.store import BaseSessionStore from kedro.io import DataCatalog from kedro.pipeline import Pipeline -from watchgod import RegExpWatcher, run_process -from kedro_viz.api import apps from kedro_viz.api.rest.responses import save_api_responses_to_fs from kedro_viz.constants import DEFAULT_HOST, DEFAULT_PORT from kedro_viz.data_access import DataAccessManager, data_access_manager @@ -56,16 +51,13 @@ def load_and_populate_data( package_name: Optional[str] = None, pipeline_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, + is_lite: bool = False, ): """Loads underlying Kedro project data and populates Kedro Viz Repositories""" # Loads data from underlying Kedro Project catalog, pipelines, session_store, stats_dict = kedro_data_loader.load_data( - path, - env, - include_hooks, - package_name, - extra_params, + path, env, include_hooks, package_name, extra_params, is_lite ) pipelines = ( @@ -78,6 +70,7 @@ def load_and_populate_data( populate_data(data_access_manager, catalog, pipelines, session_store, stats_dict) +# pylint: disable=too-many-locals def run_server( host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, @@ -90,6 +83,7 @@ def run_server( include_hooks: bool = False, package_name: Optional[str] = None, extra_params: Optional[Dict[str, Any]] = None, + is_lite: bool = False, ): # pylint: disable=redefined-outer-name """Run a uvicorn server with a FastAPI app that either launches API response data from a file or from reading data from a real Kedro project. @@ -112,18 +106,21 @@ def run_server( for underlying KedroContext. If specified, will update (and therefore take precedence over) the parameters retrieved from the project configuration. + is_lite: A flag to run Kedro-Viz in lite mode. """ + # Importing below dependencies inside `run_server` to avoid ImportError + # when calling `load_and_populate_data` from VSCode + + import fsspec # pylint: disable=C0415 + import uvicorn # pylint: disable=C0415 + + from kedro_viz.api import apps # pylint: disable=C0415 path = Path(project_path) if project_path else Path.cwd() if load_file is None: load_and_populate_data( - path, - env, - include_hooks, - package_name, - pipeline_name, - extra_params, + path, env, include_hooks, package_name, pipeline_name, extra_params, is_lite ) # [TODO: As we can do this with `kedro viz build`, # we need to shift this feature outside of kedro viz run] @@ -142,6 +139,9 @@ def run_server( if __name__ == "__main__": # pragma: no cover import argparse + import multiprocessing + + from watchgod import RegExpWatcher, run_process parser = argparse.ArgumentParser(description="Launch a development viz server") parser.add_argument("project_path", help="Path to a Kedro project") diff --git a/package/pyproject.toml b/package/pyproject.toml index 1c4e4ca1f0..7c39412920 100644 --- a/package/pyproject.toml +++ b/package/pyproject.toml @@ -36,7 +36,7 @@ azure = ["adlfs>=2021.4"] gcp = ["gcsfs>=2021.4"] [project.entry-points."kedro.global_commands"] -kedro-viz = "kedro_viz.launchers.cli:viz_cli" +kedro-viz = "kedro_viz.launchers.cli.main:viz_cli" [project.entry-points."kedro.line_magic"] line_magic = "kedro_viz.launchers.jupyter:run_viz" diff --git a/package/test_requirements.txt b/package/test_requirements.txt index 0a7c5e4ed1..14741ab0ea 100644 --- a/package/test_requirements.txt +++ b/package/test_requirements.txt @@ -10,12 +10,12 @@ boto3~=1.34 flake8~=7.1 isort~=5.11 matplotlib~=3.9 -mypy~=1.10 +mypy~=1.11 moto~=5.0.9 psutil==5.9.6 # same as Kedro for now -pylint~=3.0 +pylint~=3.2 pylint-pydantic>=0.3.0 -pytest~=8.1 +pytest~=8.3 pytest-asyncio~=0.21 pytest-mock~=3.14 pytest-cov~=5.0 diff --git a/package/tests/test_api/test_rest/test_responses.py b/package/tests/test_api/test_rest/test_responses.py index c7aad3ff0e..3f75904404 100644 --- a/package/tests/test_api/test_rest/test_responses.py +++ b/package/tests/test_api/test_rest/test_responses.py @@ -1,5 +1,5 @@ # pylint: disable=too-many-lines -import logging +import json import operator from pathlib import Path from typing import Any, Dict, Iterable, List @@ -12,8 +12,8 @@ from kedro_viz.api import apps from kedro_viz.api.rest.responses import ( EnhancedORJSONResponse, - PackageCompatibilityAPIResponse, - get_package_compatibilities_response, + get_kedro_project_json_data, + get_metadata_response, save_api_main_response_to_fs, save_api_node_response_to_fs, save_api_pipeline_response_to_fs, @@ -21,6 +21,7 @@ write_api_response_to_fs, ) from kedro_viz.models.flowchart import TaskNode +from kedro_viz.models.metadata import Metadata def _is_dict_list(collection: Any) -> bool: @@ -637,7 +638,7 @@ def test_task_node_metadata(self, client): assert metadata["outputs"] == ["model_inputs"] assert ( metadata["run_command"] - == "kedro run --to-nodes=uk.data_processing.process_data" + == "kedro run --to-nodes='uk.data_processing.process_data'" ) assert str(Path("package/tests/conftest.py")) in metadata["filepath"] @@ -833,6 +834,28 @@ def test_get_non_existing_pipeline(self, client): assert response.status_code == 404 +class TestAppMetadata: + def test_get_metadata_response(self, mocker): + mock_get_compat = mocker.patch( + "kedro_viz.api.rest.responses.get_package_compatibilities", + return_value="mocked_compatibilities", + ) + mock_set_compat = mocker.patch( + "kedro_viz.api.rest.responses.Metadata.set_package_compatibilities" + ) + + response = get_metadata_response() + + # Assert get_package_compatibilities was called + mock_get_compat.assert_called_once() + + # Assert set_package_compatibilities was called with the mocked compatibilities + mock_set_compat.assert_called_once_with("mocked_compatibilities") + + # Assert the function returns the Metadata instance + assert isinstance(response, Metadata) + + class TestAPIAppFromFile: def test_api_app_from_json_file_main_api(self): filepath = str(Path(__file__).parent.parent) @@ -849,82 +872,6 @@ def test_api_app_from_json_file_index(self): assert response.status_code == 200 -class TestPackageCompatibilities: - @pytest.mark.parametrize( - "package_name, package_version, package_requirements, expected_compatibility_response", - [ - ( - "fsspec", - "2023.9.1", - {"fsspec": {"min_compatible_version": "2023.0.0"}}, - True, - ), - ( - "fsspec", - "2023.9.1", - {"fsspec": {"min_compatible_version": "2024.0.0"}}, - False, - ), - ( - "kedro-datasets", - "2.1.0", - {"kedro-datasets": {"min_compatible_version": "2.1.0"}}, - True, - ), - ( - "kedro-datasets", - "1.8.0", - {"kedro-datasets": {"min_compatible_version": "2.1.0"}}, - False, - ), - ], - ) - def test_get_package_compatibilities_response( - self, - package_name, - package_version, - package_requirements, - expected_compatibility_response, - mocker, - ): - mocker.patch( - "kedro_viz.api.rest.responses.get_package_version", - return_value=package_version, - ) - response = get_package_compatibilities_response(package_requirements) - - for package_response in response: - assert package_response.package_name == package_name - assert package_response.package_version == package_version - assert package_response.is_compatible is expected_compatibility_response - - def test_get_package_compatibilities_exception_response(self, caplog): - mock_package_requirement = { - "random-package": { - "min_compatible_version": "1.0.0", - "warning_message": "random-package is not available", - } - } - - with caplog.at_level(logging.WARNING): - response = get_package_compatibilities_response(mock_package_requirement) - - assert len(caplog.records) == 1 - - record = caplog.records[0] - - assert record.levelname == "WARNING" - assert ( - mock_package_requirement["random-package"]["warning_message"] - in record.message - ) - - expected_response = PackageCompatibilityAPIResponse( - package_name="random-package", package_version="0.0.0", is_compatible=False - ) - assert response == [expected_response] - - class TestEnhancedORJSONResponse: @pytest.mark.parametrize( "content, expected", @@ -964,6 +911,27 @@ def test_write_api_response_to_fs( mockremote_fs.open.assert_called_once_with(file_path, "wb") mock_encode_to_human_readable.assert_called_once() + def test_get_kedro_project_json_data(self, mocker): + expected_json_data = {"key": "value"} + encoded_response = json.dumps(expected_json_data).encode("utf-8") + + mock_get_default_response = mocker.patch( + "kedro_viz.api.rest.responses.get_default_response", + return_value={"key": "value"}, + ) + mock_get_encoded_response = mocker.patch( + "kedro_viz.api.rest.responses.get_encoded_response", + return_value=encoded_response, + ) + + json_data = get_kedro_project_json_data() + + mock_get_default_response.assert_called_once() + mock_get_encoded_response.assert_called_once_with( + mock_get_default_response.return_value + ) + assert json_data == expected_json_data + def test_save_api_main_response_to_fs(self, mocker): expected_default_response = {"test": "json"} main_path = "/main" diff --git a/package/tests/test_api/test_rest/test_router.py b/package/tests/test_api/test_rest/test_router.py index e3333fea15..d84f1ce0f2 100644 --- a/package/tests/test_api/test_rest/test_router.py +++ b/package/tests/test_api/test_rest/test_router.py @@ -47,84 +47,46 @@ def test_deploy_kedro_viz( ( None, 200, - [ - { - "package_name": "fsspec", - "package_version": "2023.9.1", - "is_compatible": True, - }, - { - "package_name": "kedro-datasets", - "package_version": "1.8.0", - "is_compatible": False, - }, - ], + { + "has_missing_dependencies": False, + "package_compatibilities": [ + { + "package_name": "fsspec", + "package_version": "2023.9.1", + "is_compatible": True, + }, + { + "package_name": "kedro-datasets", + "package_version": "1.8.0", + "is_compatible": False, + }, + ], + }, ), ( Exception, 500, - {"message": "Failed to get package compatibility info"}, + {"message": "Failed to get app metadata"}, ), ], ) -def test_get_package_compatibilities( +def test_metadata( client, exception_type, expected_status_code, expected_response, mocker ): # Mock the function that may raise an exception if exception_type is None: - mocker.patch( - "kedro_viz.api.rest.router.get_package_compatibilities_response", + mock_get_metadata_response = mocker.patch( + "kedro_viz.api.rest.router.get_metadata_response", return_value=expected_response, ) else: - mocker.patch( - "kedro_viz.api.rest.router.get_package_compatibilities_response", + mock_get_metadata_response = mocker.patch( + "kedro_viz.api.rest.router.get_metadata_response", side_effect=exception_type("Test Exception"), ) - response = client.get("/api/package-compatibilities") + response = client.get("/api/metadata") + mock_get_metadata_response.assert_called_once() assert response.status_code == expected_status_code assert response.json() == expected_response - - -def test_update_preferences_success(client, mocker): - mocker.patch( - "kedro_viz.api.rest.responses.DataNodeMetadata.set_is_all_previews_enabled" - ) - response = client.post("api/preferences", json={"showDatasetPreviews": True}) - - assert response.status_code == 200 - assert response.json() == {"message": "Preferences updated successfully"} - - -def test_update_preferences_failure(client, mocker): - mocker.patch( - "kedro_viz.api.rest.responses.DataNodeMetadata.set_is_all_previews_enabled", - side_effect=Exception("Test Exception"), - ) - response = client.post("api/preferences", json={"showDatasetPreviews": True}) - - assert response.status_code == 500 - assert response.json() == {"message": "Failed to update preferences"} - - -def test_get_preferences_success(client, mocker): - mocker.patch( - "kedro_viz.api.rest.responses.DataNodeMetadata", is_all_previews_enabled=True - ) - response = client.get("/api/preferences") - - assert response.status_code == 200 - assert response.json() == {"showDatasetPreviews": True} - - -def test_get_preferences_failure(client, mocker): - mocker.patch( - "kedro_viz.api.rest.responses.DataNodeMetadata.is_all_previews_enabled", - side_effect=Exception("Test Exception"), - ) - response = client.get("/api/preferences") - - assert response.status_code == 500 - assert response.json() == {"message": "Failed to fetch preferences"} diff --git a/package/tests/test_api/test_rest/test_utils.py b/package/tests/test_api/test_rest/test_utils.py new file mode 100644 index 0000000000..32a69cc371 --- /dev/null +++ b/package/tests/test_api/test_rest/test_utils.py @@ -0,0 +1,92 @@ +import logging + +import pytest + +from kedro_viz.api.rest.utils import get_package_compatibilities +from kedro_viz.models.metadata import PackageCompatibility + +logger = logging.getLogger(__name__) + + +@pytest.mark.parametrize( + "package_name, package_version, package_requirements, expected_compatibility_response", + [ + ( + "fsspec", + "2023.9.1", + {"fsspec": {"min_compatible_version": "2023.0.0"}}, + True, + ), + ( + "fsspec", + "2023.9.1", + {"fsspec": {"min_compatible_version": "2024.0.0"}}, + False, + ), + ( + "kedro-datasets", + "2.1.0", + {"kedro-datasets": {"min_compatible_version": "2.1.0"}}, + True, + ), + ( + "kedro-datasets", + "1.8.0", + {"kedro-datasets": {"min_compatible_version": "2.1.0"}}, + False, + ), + ], +) +def test_get_package_compatibilities( + package_name, + package_version, + package_requirements, + expected_compatibility_response, + mocker, +): + mocker.patch( + "kedro_viz.api.rest.utils.get_package_version", + return_value=package_version, + ) + mocker.patch( + "kedro_viz.api.rest.utils.PACKAGE_REQUIREMENTS", + package_requirements, + ) + + response = get_package_compatibilities() + + for package_response in response: + assert package_response.package_name == package_name + assert package_response.package_version == package_version + assert package_response.is_compatible is expected_compatibility_response + + +def test_get_package_compatibilities_exception_response(caplog, mocker): + mock_package_requirement = { + "random-package": { + "min_compatible_version": "1.0.0", + "warning_message": "random-package is not available", + } + } + mocker.patch( + "kedro_viz.api.rest.utils.PACKAGE_REQUIREMENTS", + mock_package_requirement, + ) + + with caplog.at_level(logging.WARNING): + response = get_package_compatibilities() + + assert len(caplog.records) == 1 + + record = caplog.records[0] + + assert record.levelname == "WARNING" + assert ( + mock_package_requirement["random-package"]["warning_message"] + in record.message + ) + + expected_response = PackageCompatibility( + package_name="random-package", package_version="0.0.0", is_compatible=False + ) + assert response == [expected_response] diff --git a/package/tests/test_data_access/test_managers.py b/package/tests/test_data_access/test_managers.py index af94785cb9..66bd08f1e9 100644 --- a/package/tests/test_data_access/test_managers.py +++ b/package/tests/test_data_access/test_managers.py @@ -3,6 +3,7 @@ import networkx as nx import pytest from kedro.io import DataCatalog, MemoryDataset +from kedro.io.core import DatasetError from kedro.pipeline import Pipeline, node from kedro.pipeline.modular_pipeline import pipeline from kedro_datasets.pandas import CSVDataset @@ -13,6 +14,7 @@ from kedro_viz.data_access.repositories.modular_pipelines import ( ModularPipelinesRepository, ) +from kedro_viz.integrations.utils import UnavailableDataset from kedro_viz.models.flowchart import ( DataNode, GraphEdge, @@ -27,25 +29,6 @@ def identity(x): return x -def assert_expected_modular_pipeline_values_for_edge_cases( - expected_modular_pipeline_tree_obj, - modular_pipeline_node_id, - data_access_manager, - modular_pipeline_tree_values, - expected_key, -): - """This asserts an `expected_key` value present in modular_pipeline_tree - that is constructed in the edge cases with the expected_modular_pipeline_tree""" - assert sorted( - list(expected_modular_pipeline_tree_obj[modular_pipeline_node_id][expected_key]) - ) == sorted( - list( - data_access_manager.nodes.get_node_by_id(node_id).name - for node_id in modular_pipeline_tree_values - ) - ) - - class TestAddCatalog: def test_add_catalog( self, @@ -378,6 +361,29 @@ def test_add_dataset_with_modular_pipeline( "uk.data_science", } + def test_add_dataset_with_unresolved_pattern( + self, + data_access_manager: DataAccessManager, + example_pipelines: Dict[str, Pipeline], + example_modular_pipelines_repo_obj, + mocker, + ): + dataset = CSVDataset(filepath="dataset.csv") + dataset_name = "companies#csv" + catalog = DataCatalog(datasets={dataset_name: dataset}) + data_access_manager.add_catalog(catalog, example_pipelines) + + with mocker.patch.object( + data_access_manager.catalog, + "get_dataset", + side_effect=DatasetError("Dataset not found"), + ): + dataset_obj = data_access_manager.add_dataset( + "my_pipeline", dataset_name, example_modular_pipelines_repo_obj + ) + + assert isinstance(dataset_obj.kedro_obj, UnavailableDataset) + def test_add_all_parameters( self, data_access_manager: DataAccessManager, diff --git a/package/tests/test_integrations/test_abstract_dataset_lite.py b/package/tests/test_integrations/test_abstract_dataset_lite.py new file mode 100644 index 0000000000..184fee0413 --- /dev/null +++ b/package/tests/test_integrations/test_abstract_dataset_lite.py @@ -0,0 +1,54 @@ +from unittest.mock import Mock, patch + +import pytest +from kedro.io.core import DatasetError + +from kedro_viz.integrations.kedro.abstract_dataset_lite import AbstractDatasetLite +from kedro_viz.integrations.utils import UnavailableDataset + + +@pytest.fixture +def filepath(tmp_path): + return (tmp_path / "some" / "dir" / "test.csv").as_posix() + + +def test_from_config_success(filepath): + with patch("kedro.io.core.AbstractDataset.from_config") as mock_from_config: + mock_dataset = Mock() + mock_from_config.return_value = mock_dataset + + # Call the method + result = AbstractDatasetLite.from_config( + name="boats", + config={"type": "pandas.CSVDataset", "filepath": filepath}, + load_version=None, + save_version=None, + ) + + # Assert that the result is the mock dataset + assert result == mock_dataset + mock_from_config.assert_called_once_with( + "boats", {"type": "pandas.CSVDataset", "filepath": filepath}, None, None + ) + + +def test_from_config_failure(): + with patch( + "kedro.io.core.AbstractDataset.from_config", + side_effect=DatasetError( + "An exception occurred when parsing config of a dataset" + ), + ) as mock_from_config: + # Call the method + result = AbstractDatasetLite.from_config( + name="boats", + config={"type": "pandas.CSVDataset", "filepath": filepath}, + load_version=None, + save_version=None, + ) + + # Assert that UnavailableDataset is returned + assert isinstance(result, UnavailableDataset) + mock_from_config.assert_called_once_with( + "boats", {"type": "pandas.CSVDataset", "filepath": filepath}, None, None + ) diff --git a/package/tests/test_integrations/test_lite_parser.py b/package/tests/test_integrations/test_lite_parser.py new file mode 100644 index 0000000000..b3ae1eedea --- /dev/null +++ b/package/tests/test_integrations/test_lite_parser.py @@ -0,0 +1,195 @@ +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from kedro_viz.integrations.kedro.lite_parser import LiteParser + + +@pytest.fixture +def sample_project_path(tmp_path): + # Create a sample directory structure + package_dir = tmp_path / "mock_spaceflights" + package_dir.mkdir() + (package_dir / "__init__.py").touch() + (package_dir / "__init__.py").write_text( + "from mock_spaceflights import data_processing\n" + "from mock_spaceflights.data_processing import create_metrics" + ) + (package_dir / "data_processing.py").write_text( + "import os\n" + "import nonexistentmodule\n" + "from . import test\n" + "from typing import Dict" + ) + return tmp_path + + +@pytest.fixture +def lite_parser(): + return LiteParser(package_name="mock_spaceflights") + + +class TestLiteParser: + def test_is_module_importable_existing_module(self, lite_parser): + assert lite_parser._is_module_importable("os") is True + + def test_is_module_importable_nonexistent_module(self, lite_parser): + assert lite_parser._is_module_importable("nonexistentmodule") is False + + def test_is_module_importable_importerror(self, lite_parser): + with patch("importlib.util.find_spec", side_effect=ImportError): + assert lite_parser._is_module_importable("nonexistentmodule") is False + + def test_is_module_importable_modulenotfounderror(self, lite_parser): + with patch("importlib.util.find_spec", side_effect=ModuleNotFoundError): + assert lite_parser._is_module_importable("nonexistentmodule") is False + + def test_is_module_importable_valueerror(self, lite_parser): + with patch("importlib.util.find_spec", side_effect=ValueError): + assert lite_parser._is_module_importable("nonexistentmodule") is False + + @pytest.mark.parametrize( + "module_name, expected_module_parts", + [ + ("sklearn", ["sklearn"]), + ( + "demo_project.pipelines.ingestion", + [ + "demo_project", + "demo_project.pipelines", + "demo_project.pipelines.ingestion", + ], + ), + ], + ) + def test_get_module_parts(self, lite_parser, module_name, expected_module_parts): + assert lite_parser._get_module_parts(module_name) == expected_module_parts + + def test_is_relative_import_found(self, lite_parser): + module_name = "kedro_project_package.pipelines.reporting.nodes" + project_file_paths = { + Path("/path/to/kedro_project_package/pipelines/reporting/nodes.py") + } + assert lite_parser._is_relative_import(module_name, project_file_paths) + + def test_relative_import_not_found(self, lite_parser): + module_name = "kedro_project_package.pipelines.reporting.nodes" + project_file_paths = { + Path("/path/to/another_project/pipelines/reporting/nodes.py") + } + assert not lite_parser._is_relative_import(module_name, project_file_paths) + + def test_relative_import_partial_match(self, lite_parser): + module_name = "kedro_project_package.pipelines" + project_file_paths = { + Path("/path/to/kedro_project_package/pipelines/reporting/nodes.py"), + Path("/path/to/kedro_project_package/pipelines/something_else.py"), + } + assert lite_parser._is_relative_import(module_name, project_file_paths) + + def test_relative_import_empty_file_paths(self, lite_parser): + module_name = "kedro_project_package.pipelines.reporting.nodes" + project_file_paths = set() + assert not lite_parser._is_relative_import(module_name, project_file_paths) + + def test_populate_missing_dependencies(self, lite_parser): + module_name = "non_importable.module.part" + missing_dependencies = set() + + lite_parser._populate_missing_dependencies(module_name, missing_dependencies) + + # The test expects the missing dependencies to + # include each part of the module name + expected_missing = { + "non_importable", + "non_importable.module", + "non_importable.module.part", + } + assert missing_dependencies == expected_missing + + def test_no_missing_dependencies(self, lite_parser, mocker): + module_name = "importable_module" + missing_dependencies = set() + mocker.patch( + "kedro_viz.integrations.kedro.lite_parser.LiteParser._is_module_importable", + return_value=True, + ) + + lite_parser._populate_missing_dependencies(module_name, missing_dependencies) + + # Since the module is importable, + # the set should remain empty + assert not missing_dependencies + + def test_partial_importability(self, lite_parser, mocker): + module_name = "importable_module.non_importable_part" + missing_dependencies = set() + mocker.patch( + "kedro_viz.integrations.kedro.lite_parser.LiteParser._is_module_importable", + side_effect=lambda part: part == "importable_module", + ) + + lite_parser._populate_missing_dependencies(module_name, missing_dependencies) + + # Only the non-importable part + # should be added to the set + expected_missing = {"importable_module.non_importable_part"} + assert missing_dependencies == expected_missing + + def test_get_unresolved_imports(self, lite_parser, sample_project_path, mocker): + file_path = Path(sample_project_path / "mock_spaceflights/data_processing.py") + mock_populate = mocker.patch( + "kedro_viz.integrations.kedro.lite_parser.LiteParser._populate_missing_dependencies" + ) + + lite_parser._get_unresolved_imports(file_path) + + # Ensure _populate_missing_dependencies was called + # with correct module names + mock_populate.assert_any_call("os", set()) + mock_populate.assert_any_call("nonexistentmodule", set()) + + def test_get_unresolved_relative_imports(self, sample_project_path, mocker): + lite_parser_obj = LiteParser() + file_path = Path(sample_project_path / "mock_spaceflights/__init__.py") + + unresolvable_imports = lite_parser_obj._get_unresolved_imports( + file_path, set(sample_project_path.rglob("*.py")) + ) + + assert len(unresolvable_imports) == 0 + + def test_create_mock_modules(self, lite_parser): + unresolved_imports = {"sklearn", "pyspark.pandas"} + mocked_modules = lite_parser.create_mock_modules(unresolved_imports) + + assert len(mocked_modules) == len(unresolved_imports) + assert "sklearn" in mocked_modules + assert "pyspark.pandas" in mocked_modules + assert isinstance(mocked_modules["sklearn"], MagicMock) + + def test_parse_non_existent_path(self, lite_parser): + assert not lite_parser.parse(Path("non/existent/path")) + assert not lite_parser.parse(Path("non/existent/path/file.py")) + + def test_file_parse(self, lite_parser, sample_project_path): + file_path = Path(sample_project_path / "mock_spaceflights/data_processing.py") + unresolved_imports = lite_parser.parse(file_path) + + assert unresolved_imports == {str(file_path): {"nonexistentmodule"}} + + def test_directory_parse(self, lite_parser, sample_project_path): + unresolved_imports = lite_parser.parse(sample_project_path) + expected_file_path = Path( + sample_project_path / "mock_spaceflights/data_processing.py" + ) + assert unresolved_imports == {str(expected_file_path): {"nonexistentmodule"}} + + def test_directory_parse_non_package_path(self, sample_project_path): + lite_parser_obj = LiteParser("mock_pyspark") + unresolvable_imports = lite_parser_obj.parse(sample_project_path) + + # ignore files in other packages if + # LiteParser is instantiated with a package_name + assert len(unresolvable_imports) == 0 diff --git a/package/tests/test_launchers/test_cli.py b/package/tests/test_launchers/test_cli.py deleted file mode 100755 index 82403cd5e7..0000000000 --- a/package/tests/test_launchers/test_cli.py +++ /dev/null @@ -1,797 +0,0 @@ -from unittest.mock import Mock, call - -import pytest -import requests -from click.testing import CliRunner -from packaging.version import parse -from watchgod import RegExpWatcher, run_process - -from kedro_viz import __version__ -from kedro_viz.constants import SHAREABLEVIZ_SUPPORTED_PLATFORMS, VIZ_DEPLOY_TIME_LIMIT -from kedro_viz.launchers import cli -from kedro_viz.launchers.utils import _PYPROJECT -from kedro_viz.server import run_server - - -@pytest.fixture -def patched_check_viz_up(mocker): - mocker.patch("kedro_viz.launchers.cli._check_viz_up", return_value=True) - - -@pytest.fixture -def patched_start_browser(mocker): - mocker.patch("kedro_viz.launchers.cli._start_browser") - - -@pytest.fixture -def mock_viz_deploy_process(mocker): - return mocker.patch("kedro_viz.launchers.cli.multiprocessing.Process") - - -@pytest.fixture -def mock_process_completed(mocker): - return mocker.patch( - "kedro_viz.launchers.cli.multiprocessing.Value", return_value=Mock() - ) - - -@pytest.fixture -def mock_exception_queue(mocker): - return mocker.patch( - "kedro_viz.launchers.cli.multiprocessing.Queue", return_value=Mock() - ) - - -@pytest.fixture -def mock_viz_load_and_deploy(mocker): - return mocker.patch("kedro_viz.launchers.cli.load_and_deploy_viz") - - -@pytest.fixture -def mock_viz_deploy_progress_timer(mocker): - return mocker.patch("kedro_viz.launchers.cli.viz_deploy_progress_timer") - - -@pytest.fixture -def mock_DeployerFactory(mocker): - return mocker.patch("kedro_viz.launchers.cli.DeployerFactory") - - -@pytest.fixture -def mock_load_and_populate_data(mocker): - return mocker.patch("kedro_viz.launchers.cli.load_and_populate_data") - - -@pytest.fixture -def mock_click_echo(mocker): - return mocker.patch("click.echo") - - -@pytest.fixture -def mock_project_path(mocker): - mock_path = "/tmp/project_path" - mocker.patch("pathlib.Path.cwd", return_value=mock_path) - return mock_path - - -@pytest.mark.parametrize( - "command_options,run_server_args", - [ - ( - ["viz"], - { - "host": "127.0.0.1", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {}, - }, - ), - ( - ["viz", "run"], - { - "host": "127.0.0.1", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {}, - }, - ), - ( - [ - "viz", - "run", - "--host", - "localhost", - ], - { - "host": "localhost", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {}, - }, - ), - ( - [ - "viz", - "run", - "--host", - "8.8.8.8", - "--port", - "4142", - "--no-browser", - "--save-file", - "save_dir", - "--pipeline", - "data_science", - "--env", - "local", - "--params", - "extra_param=param", - ], - { - "host": "8.8.8.8", - "port": 4142, - "load_file": None, - "save_file": "save_dir", - "pipeline_name": "data_science", - "env": "local", - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {"extra_param": "param"}, - }, - ), - ( - [ - "viz", - "run", - "--host", - "8.8.8.8", - "--port", - "4142", - "--no-browser", - "--save-file", - "save_dir", - "-p", - "data_science", - "-e", - "local", - "--params", - "extra_param=param", - ], - { - "host": "8.8.8.8", - "port": 4142, - "load_file": None, - "save_file": "save_dir", - "pipeline_name": "data_science", - "env": "local", - "project_path": "testPath", - "autoreload": False, - "include_hooks": False, - "package_name": None, - "extra_params": {"extra_param": "param"}, - }, - ), - ( - ["viz", "run", "--include-hooks"], - { - "host": "127.0.0.1", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "project_path": "testPath", - "autoreload": False, - "include_hooks": True, - "package_name": None, - "extra_params": {}, - }, - ), - ], -) -def test_kedro_viz_command_run_server( - command_options, - run_server_args, - mocker, - patched_check_viz_up, - patched_start_browser, -): - process_init = mocker.patch("multiprocessing.Process") - runner = CliRunner() - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", - return_value=run_server_args["project_path"], - ) - - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, command_options) - - process_init.assert_called_once_with( - target=run_server, daemon=False, kwargs={**run_server_args} - ) - assert run_server_args["port"] in cli._VIZ_PROCESSES - - -def test_kedro_viz_command_should_log_project_not_found( - mocker, mock_project_path, mock_click_echo -): - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch("kedro_viz.launchers.cli._find_kedro_project", return_value=None) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run"]) - - mock_click_echo_calls = [ - call( - "\x1b[31mERROR: Failed to start Kedro-Viz : " - "Could not find the project configuration " - f"file '{_PYPROJECT}' at '{mock_project_path}'. \x1b[0m" - ) - ] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_kedro_viz_command_should_log_outdated_version( - mocker, mock_http_response, mock_click_echo, mock_project_path -): - installed_version = parse(__version__) - mock_version = f"{installed_version.major + 1}.0.0" - requests_get = mocker.patch("requests.get") - requests_get.return_value = mock_http_response( - data={"info": {"version": mock_version}} - ) - - mocker.patch("kedro_viz.server.run_server") - - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path - ) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run"]) - - mock_click_echo_calls = [ - call( - "\x1b[33mWARNING: You are using an old version of Kedro Viz. " - f"You are using version {installed_version}; " - f"however, version {mock_version} is now available.\n" - "You should consider upgrading via the `pip install -U kedro-viz` command.\n" - "You can view the complete changelog at " - "https://github.com/kedro-org/kedro-viz/releases.\x1b[0m" - ) - ] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_kedro_viz_command_should_not_log_latest_version( - mocker, mock_http_response, mock_click_echo, mock_project_path -): - requests_get = mocker.patch("requests.get") - requests_get.return_value = mock_http_response( - data={"info": {"version": str(parse(__version__))}} - ) - - mocker.patch("kedro_viz.server.run_server") - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path - ) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run"]) - - mock_click_echo_calls = [call("\x1b[32mStarting Kedro Viz ...\x1b[0m")] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_kedro_viz_command_should_not_log_if_pypi_is_down( - mocker, mock_http_response, mock_click_echo, mock_project_path -): - requests_get = mocker.patch("requests.get") - requests_get.side_effect = requests.exceptions.RequestException("PyPI is down") - - mocker.patch("kedro_viz.server.run_server") - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path - ) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run"]) - - mock_click_echo_calls = [call("\x1b[32mStarting Kedro Viz ...\x1b[0m")] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_kedro_viz_command_with_autoreload( - mocker, patched_check_viz_up, patched_start_browser, mock_project_path -): - process_init = mocker.patch("multiprocessing.Process") - - # Reduce the timeout argument from 600 to 1 to make test run faster. - mocker.patch("kedro_viz.launchers.cli._wait_for.__defaults__", (True, 1, True, 1)) - # Mock finding kedro project - mocker.patch( - "kedro_viz.launchers.cli._find_kedro_project", return_value=mock_project_path - ) - runner = CliRunner() - with runner.isolated_filesystem(): - runner.invoke(cli.viz_cli, ["viz", "run", "--autoreload"]) - - run_process_kwargs = { - "path": mock_project_path, - "target": run_server, - "kwargs": { - "host": "127.0.0.1", - "port": 4141, - "load_file": None, - "save_file": None, - "pipeline_name": None, - "env": None, - "autoreload": True, - "project_path": mock_project_path, - "include_hooks": False, - "package_name": None, - "extra_params": {}, - }, - "watcher_cls": RegExpWatcher, - "watcher_kwargs": {"re_files": "^.*(\\.yml|\\.yaml|\\.py|\\.json)$"}, - } - - process_init.assert_called_once_with( - target=run_process, daemon=False, kwargs={**run_process_kwargs} - ) - assert run_process_kwargs["kwargs"]["port"] in cli._VIZ_PROCESSES - - -def test_viz_command_group(mocker, mock_click_echo): - runner = CliRunner() - - with runner.isolated_filesystem(): - result = runner.invoke(cli.viz_cli, ["viz", "--help"]) - - assert result.output == ( - "Usage: Kedro-Viz viz [OPTIONS] COMMAND [ARGS]...\n" - "\n" - " Visualise a Kedro pipeline using Kedro viz.\n" - "\n" - "Options:\n" - " --help Show this message and exit.\n" - "\n" - "Commands:\n" - " run* Launch local Kedro Viz instance\n" - " build Create build directory of local Kedro Viz instance with Kedro...\n" - " deploy Deploy and host Kedro Viz on provided platform\n" - ) - - -@pytest.mark.parametrize( - "command_options, deployer_args", - [ - ( - [ - "viz", - "deploy", - "--platform", - "azure", - "--endpoint", - "https://example-bucket.web.core.windows.net", - "--bucket-name", - "example-bucket", - ], - { - "platform": "azure", - "endpoint": "https://example-bucket.web.core.windows.net", - "bucket_name": "example-bucket", - }, - ), - ( - [ - "viz", - "deploy", - "--platform", - "aws", - "--endpoint", - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "--bucket-name", - "example-bucket", - ], - { - "platform": "aws", - "endpoint": "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "bucket_name": "example-bucket", - }, - ), - ( - [ - "viz", - "deploy", - "--platform", - "gcp", - "--endpoint", - "http://34.120.87.227/", - "--bucket-name", - "example-bucket", - ], - { - "platform": "gcp", - "endpoint": "http://34.120.87.227/", - "bucket_name": "example-bucket", - }, - ), - ( - [ - "viz", - "deploy", - "--platform", - "gcp", - "--endpoint", - "http://34.120.87.227/", - "--bucket-name", - "example-bucket", - "--include-hooks", - ], - { - "platform": "gcp", - "endpoint": "http://34.120.87.227/", - "bucket_name": "example-bucket", - "include_hooks": True, - }, - ), - ( - [ - "viz", - "deploy", - "--platform", - "aws", - "--endpoint", - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "--bucket-name", - "example-bucket", - "--include-previews", - ], - { - "platform": "aws", - "endpoint": "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "bucket_name": "example-bucket", - "preview": True, - }, - ), - ], -) -def test_viz_deploy_valid_endpoint_and_bucket(command_options, deployer_args, mocker): - runner = CliRunner() - mocker.patch("fsspec.filesystem") - create_shareableviz_process_mock = mocker.patch( - "kedro_viz.launchers.cli.create_shareableviz_process" - ) - - with runner.isolated_filesystem(): - result = runner.invoke(cli.viz_cli, command_options) - - assert result.exit_code == 0 - - create_shareableviz_process_mock.assert_called_once_with( - deployer_args.get("platform"), - deployer_args.get("preview", False), - deployer_args.get("endpoint"), - deployer_args.get("bucket_name"), - deployer_args.get("include_hooks", False), - ) - - -def test_viz_deploy_invalid_platform(mocker, mock_click_echo): - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke( - cli.viz_cli, - [ - "viz", - "deploy", - "--platform", - "random", - "--endpoint", - "", - "--bucket-name", - "example-bucket", - ], - ) - - assert result.exit_code == 0 - mock_click_echo_calls = [ - call( - "\x1b[31mERROR: Invalid platform specified. Kedro-Viz supports \n" - f"the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}\x1b[0m" - ) - ] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -def test_viz_deploy_invalid_endpoint(mocker, mock_click_echo): - runner = CliRunner() - with runner.isolated_filesystem(): - result = runner.invoke( - cli.viz_cli, - [ - "viz", - "deploy", - "--platform", - "aws", - "--endpoint", - "", - "--bucket-name", - "example-bucket", - ], - ) - - assert result.exit_code == 0 - mock_click_echo_calls = [ - call( - "\x1b[31mERROR: Invalid endpoint specified. If you are looking for platform \n" - "agnostic shareable viz solution, please use the `kedro viz build` command\x1b[0m" - ) - ] - - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -@pytest.mark.parametrize( - "command_options, build_args", - [ - ( - [ - "viz", - "build", - ], - { - "platform": "local", - }, - ), - ( - ["viz", "build", "--include-hooks"], - {"platform": "local", "include_hooks": True}, - ), - ( - ["viz", "build", "--include-previews"], - {"platform": "local", "preview": True}, - ), - ], -) -def test_successful_build_with_existing_static_files( - command_options, build_args, mocker -): - runner = CliRunner() - mocker.patch("fsspec.filesystem") - create_shareableviz_process_mock = mocker.patch( - "kedro_viz.launchers.cli.create_shareableviz_process" - ) - - with runner.isolated_filesystem(): - result = runner.invoke(cli.viz_cli, command_options) - - assert result.exit_code == 0 - - create_shareableviz_process_mock.assert_called_once_with( - build_args.get("platform"), - build_args.get("preview", False), - include_hooks=build_args.get("include_hooks", False), - ) - - -@pytest.mark.parametrize( - "platform, is_all_previews_enabled, endpoint, bucket_name," - "include_hooks, process_completed_value", - [ - ( - "azure", - True, - "https://example-bucket.web.core.windows.net", - "example-bucket", - True, - 1, - ), - ( - "aws", - True, - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "example-bucket", - True, - 1, - ), - ( - "gcp", - False, - "http://34.120.87.227/", - "example-bucket", - False, - 1, - ), - ("local", False, None, None, False, 1), - ( - "azure", - True, - "https://example-bucket.web.core.windows.net", - "example-bucket", - False, - 0, - ), - ( - "aws", - False, - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "example-bucket", - False, - 0, - ), - ( - "gcp", - True, - "http://34.120.87.227/", - "example-bucket", - True, - 0, - ), - ("local", True, None, None, True, 0), - ], -) -def test_create_shareableviz_process( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - process_completed_value, - mock_viz_deploy_process, - mock_process_completed, - mock_exception_queue, - mock_viz_load_and_deploy, - mock_viz_deploy_progress_timer, - mock_click_echo, -): - mock_process_completed.return_value.value = process_completed_value - cli.create_shareableviz_process( - platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks - ) - - # Assert the mocks were called as expected - mock_viz_deploy_process.assert_called_once_with( - target=mock_viz_load_and_deploy, - args=( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - None, - mock_process_completed.return_value, - mock_exception_queue.return_value, - ), - ) - mock_viz_deploy_process.return_value.start.assert_called_once() - mock_viz_deploy_progress_timer.assert_called_once_with( - mock_process_completed.return_value, VIZ_DEPLOY_TIME_LIMIT - ) - mock_viz_deploy_process.return_value.terminate.assert_called_once() - - if process_completed_value: - if platform != "local": - msg = ( - "\x1b[32m\u2728 Success! Kedro Viz has been deployed on " - f"{platform.upper()}. " - "It can be accessed at :\n" - f"{endpoint}\x1b[0m" - ) - else: - msg = ( - "\x1b[32m✨ Success! Kedro-Viz build files have been " - "added to the `build` directory.\x1b[0m" - ) - else: - msg = ( - "\x1b[31mTIMEOUT ERROR: Failed to build/deploy Kedro-Viz " - f"as the process took more than {VIZ_DEPLOY_TIME_LIMIT} seconds. " - "Please try again later.\x1b[0m" - ) - - mock_click_echo_calls = [call(msg)] - mock_click_echo.assert_has_calls(mock_click_echo_calls) - - -@pytest.mark.parametrize( - "platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks, package_name", - [ - ( - "azure", - False, - "https://example-bucket.web.core.windows.net", - "example-bucket", - False, - "demo_project", - ), - ( - "aws", - True, - "http://example-bucket.s3-website.us-east-2.amazonaws.com/", - "example-bucket", - True, - "demo_project", - ), - ("gcp", True, "http://34.120.87.227/", "example-bucket", False, "demo_project"), - ("local", False, None, None, True, "demo_project"), - ], -) -def test_load_and_deploy_viz_success( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - package_name, - mock_DeployerFactory, - mock_load_and_populate_data, - mock_process_completed, - mock_exception_queue, - mock_click_echo, - mock_project_path, -): - deployer_mock = mock_DeployerFactory.create_deployer.return_value - - cli.load_and_deploy_viz( - platform, - is_all_previews_enabled, - endpoint, - bucket_name, - include_hooks, - package_name, - mock_process_completed, - mock_exception_queue, - ) - - mock_load_and_populate_data.assert_called_once_with( - mock_project_path, include_hooks=include_hooks, package_name=package_name - ) - mock_DeployerFactory.create_deployer.assert_called_once_with( - platform, endpoint, bucket_name - ) - deployer_mock.deploy.assert_called_once_with(is_all_previews_enabled) - mock_click_echo.echo.assert_not_called() diff --git a/package/tests/test_launchers/test_cli/test_build.py b/package/tests/test_launchers/test_cli/test_build.py new file mode 100644 index 0000000000..8918b2a21f --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_build.py @@ -0,0 +1,49 @@ +import pytest +from click.testing import CliRunner + +from kedro_viz import __version__ +from kedro_viz.launchers.cli import main + + +class TestCliBuildViz: + @pytest.mark.parametrize( + "command_options, build_args", + [ + ( + [ + "viz", + "build", + ], + { + "platform": "local", + }, + ), + ( + ["viz", "build", "--include-hooks"], + {"platform": "local", "include_hooks": True}, + ), + ( + ["viz", "build", "--include-previews"], + {"platform": "local", "preview": True}, + ), + ], + ) + def test_successful_build_with_existing_static_files( + self, command_options, build_args, mocker + ): + runner = CliRunner() + mocker.patch("fsspec.filesystem") + create_shareableviz_process_mock = mocker.patch( + "kedro_viz.launchers.cli.utils.create_shareableviz_process" + ) + + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, command_options) + + assert result.exit_code == 0 + + create_shareableviz_process_mock.assert_called_once_with( + build_args.get("platform"), + build_args.get("preview", False), + include_hooks=build_args.get("include_hooks", False), + ) diff --git a/package/tests/test_launchers/test_cli/test_deploy.py b/package/tests/test_launchers/test_cli/test_deploy.py new file mode 100644 index 0000000000..180fe96717 --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_deploy.py @@ -0,0 +1,185 @@ +from unittest.mock import call + +import pytest +from click.testing import CliRunner + +from kedro_viz import __version__ +from kedro_viz.constants import SHAREABLEVIZ_SUPPORTED_PLATFORMS +from kedro_viz.launchers.cli import main + + +@pytest.fixture +def mock_click_echo(mocker): + return mocker.patch("click.echo") + + +class TestCliDeployViz: + @pytest.mark.parametrize( + "command_options, deployer_args", + [ + ( + [ + "viz", + "deploy", + "--platform", + "azure", + "--endpoint", + "https://example-bucket.web.core.windows.net", + "--bucket-name", + "example-bucket", + ], + { + "platform": "azure", + "endpoint": "https://example-bucket.web.core.windows.net", + "bucket_name": "example-bucket", + }, + ), + ( + [ + "viz", + "deploy", + "--platform", + "aws", + "--endpoint", + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "--bucket-name", + "example-bucket", + ], + { + "platform": "aws", + "endpoint": "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "bucket_name": "example-bucket", + }, + ), + ( + [ + "viz", + "deploy", + "--platform", + "gcp", + "--endpoint", + "http://34.120.87.227/", + "--bucket-name", + "example-bucket", + ], + { + "platform": "gcp", + "endpoint": "http://34.120.87.227/", + "bucket_name": "example-bucket", + }, + ), + ( + [ + "viz", + "deploy", + "--platform", + "gcp", + "--endpoint", + "http://34.120.87.227/", + "--bucket-name", + "example-bucket", + "--include-hooks", + ], + { + "platform": "gcp", + "endpoint": "http://34.120.87.227/", + "bucket_name": "example-bucket", + "include_hooks": True, + }, + ), + ( + [ + "viz", + "deploy", + "--platform", + "aws", + "--endpoint", + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "--bucket-name", + "example-bucket", + "--include-previews", + ], + { + "platform": "aws", + "endpoint": "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "bucket_name": "example-bucket", + "preview": True, + }, + ), + ], + ) + def test_viz_deploy_valid_endpoint_and_bucket( + self, command_options, deployer_args, mocker + ): + runner = CliRunner() + mocker.patch("fsspec.filesystem") + create_shareableviz_process_mock = mocker.patch( + "kedro_viz.launchers.cli.utils.create_shareableviz_process" + ) + + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, command_options) + + assert result.exit_code == 0 + + create_shareableviz_process_mock.assert_called_once_with( + deployer_args.get("platform"), + deployer_args.get("preview", False), + deployer_args.get("endpoint"), + deployer_args.get("bucket_name"), + deployer_args.get("include_hooks", False), + ) + + def test_viz_deploy_invalid_platform(self, mock_click_echo): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + main.viz_cli, + [ + "viz", + "deploy", + "--platform", + "random", + "--endpoint", + "", + "--bucket-name", + "example-bucket", + ], + ) + + assert result.exit_code == 0 + mock_click_echo_calls = [ + call( + "\x1b[31mERROR: Invalid platform specified. Kedro-Viz supports \n" + f"the following platforms - {*SHAREABLEVIZ_SUPPORTED_PLATFORMS,}\x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_viz_deploy_invalid_endpoint(self, mock_click_echo): + runner = CliRunner() + with runner.isolated_filesystem(): + result = runner.invoke( + main.viz_cli, + [ + "viz", + "deploy", + "--platform", + "aws", + "--endpoint", + "", + "--bucket-name", + "example-bucket", + ], + ) + + assert result.exit_code == 0 + mock_click_echo_calls = [ + call( + "\x1b[31mERROR: Invalid endpoint specified. If you are looking for platform \n" + "agnostic shareable viz solution, please use the `kedro viz build` command\x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) diff --git a/package/tests/test_launchers/test_cli/test_lazy_default_group.py b/package/tests/test_launchers/test_cli/test_lazy_default_group.py new file mode 100644 index 0000000000..5148d4d76d --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_lazy_default_group.py @@ -0,0 +1,82 @@ +from unittest.mock import MagicMock, patch + +import pytest +from click import Context, UsageError + +from kedro_viz.launchers.cli.lazy_default_group import LazyDefaultGroup + + +@pytest.fixture +def lazy_default_group(): + """Fixture for LazyDefaultGroup.""" + return LazyDefaultGroup( + name="viz_cli_group", + lazy_subcommands={ + "run": "kedro_viz.launchers.cli.run.run", + "build": "kedro_viz.launchers.cli.build.build", + }, + default="build", + default_if_no_args=True, + ) + + +def test_lazy_loading(lazy_default_group): + """Test that lazy loading of a command works.""" + with patch("importlib.import_module") as mock_import_module: + mock_command = MagicMock() + mock_import_module.return_value.run = mock_command + + cmd = lazy_default_group.get_command(Context(lazy_default_group), "run") + + assert cmd == mock_command + mock_import_module.assert_called_once_with("kedro_viz.launchers.cli.run") + + +def test_list_commands(lazy_default_group): + """Test that the list of commands is correctly returned.""" + commands = lazy_default_group.list_commands(Context(lazy_default_group)) + assert commands == ["build", "run"] + + +def test_default_command_if_no_args(lazy_default_group): + """Test that the default command is invoked when no args are passed.""" + ctx = Context(lazy_default_group) + args = [] + + lazy_default_group.parse_args(ctx, args) + + # Assert that the default command is used + assert args == ["build"] + + +def test_resolve_command_with_valid_command(lazy_default_group): + """Test resolving a valid command.""" + ctx = Context(lazy_default_group) + cmd_name, cmd, args = lazy_default_group.resolve_command(ctx, ["run"]) + assert cmd_name == "run" + assert cmd is not None + + +def test_resolve_command_with_invalid_command(lazy_default_group): + """Test resolving an invalid command falls back to default.""" + ctx = Context(lazy_default_group) + + # When an invalid command is given, the default command should be used + cmd_name, cmd, args = lazy_default_group.resolve_command(ctx, ["invalid"]) + assert cmd_name == "build" + assert cmd is not None + + +def test_resolve_command_raises_usage_error_when_no_default(lazy_default_group): + """Test that UsageError is raised when an invalid command is given and no default is set.""" + lazy_default_group.default_cmd_name = None # Remove the default command + + ctx = Context(lazy_default_group) + with pytest.raises(UsageError): + lazy_default_group.resolve_command(ctx, ["invalid"]) + + +def test_init_raises_value_error_on_ignore_unknown_options(): + """Test that ValueError is raised when ignore_unknown_options is False.""" + with pytest.raises(ValueError): + LazyDefaultGroup(ignore_unknown_options=False) diff --git a/package/tests/test_launchers/test_cli/test_main.py b/package/tests/test_launchers/test_cli/test_main.py new file mode 100644 index 0000000000..a1546faa6e --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_main.py @@ -0,0 +1,75 @@ +import pytest +from click.testing import CliRunner + +from kedro_viz.launchers.cli import main +from kedro_viz.launchers.cli.lazy_default_group import LazyDefaultGroup + + +@pytest.fixture(scope="class") +def runner(): + return CliRunner() + + +class TestCLIMain: + def test_viz_cli_group(self): + assert len(main.viz_cli.list_commands(None)) == 1 + assert len(main.viz.list_commands(None)) == 3 + + assert main.viz_cli.list_commands(None) == ["viz"] + assert main.viz.list_commands(None) == ["build", "deploy", "run"] + + assert main.viz_cli.get_command(None, "random") is None + assert main.viz_cli.get_command(None, "viz") is not None + assert main.viz.get_command(None, "run") is not None + + assert isinstance(main.viz_cli.get_command(None, "viz"), LazyDefaultGroup) + + def test_viz_help(self, runner): + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, ["viz", "--help"]) + + assert result.output == ( + "Usage: Kedro-Viz viz [OPTIONS] COMMAND [ARGS]...\n" + "\n" + " Visualise a Kedro pipeline using Kedro viz.\n" + "\n" + "Options:\n" + " --help Show this message and exit.\n" + "\n" + "Commands:\n" + " build Create build directory of local Kedro Viz instance with Kedro...\n" + " deploy Deploy and host Kedro Viz on provided platform\n" + " run Launch local Kedro Viz instance\n" + ) + + def test_viz_run_help(self, runner): + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, ["viz", "run", "--help"]) + + assert result.exit_code == 0 + assert "Launch local Kedro Viz instance" in result.output + assert "invalid-option" not in result.output + assert "--host" in result.output + + def test_viz_build_help(self, runner): + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, ["viz", "build", "--help"]) + + assert result.exit_code == 0 + assert ( + "Create build directory of local Kedro Viz instance with Kedro project data" + in result.output + ) + assert "invalid-option" not in result.output + assert "--include-hooks" in result.output + assert "--include-previews" in result.output + + def test_viz_deploy_help(self, runner): + with runner.isolated_filesystem(): + result = runner.invoke(main.viz_cli, ["viz", "deploy", "--help"]) + + assert result.exit_code == 0 + assert "Deploy and host Kedro Viz on provided platform" in result.output + assert "invalid-option" not in result.output + assert "--platform" in result.output + assert "--bucket-name" in result.output diff --git a/package/tests/test_launchers/test_cli/test_run.py b/package/tests/test_launchers/test_cli/test_run.py new file mode 100644 index 0000000000..b2d5c59b39 --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_run.py @@ -0,0 +1,384 @@ +from unittest.mock import call + +import pytest +import requests +from click.testing import CliRunner +from packaging.version import parse +from watchgod import RegExpWatcher, run_process + +from kedro_viz import __version__ +from kedro_viz.launchers.cli import main +from kedro_viz.launchers.cli.run import _VIZ_PROCESSES +from kedro_viz.launchers.utils import _PYPROJECT +from kedro_viz.server import run_server + + +@pytest.fixture +def patched_check_viz_up(mocker): + mocker.patch("kedro_viz.launchers.utils._check_viz_up", return_value=True) + + +@pytest.fixture +def patched_start_browser(mocker): + mocker.patch("kedro_viz.launchers.utils._start_browser") + + +@pytest.fixture +def mock_click_echo(mocker): + return mocker.patch("click.echo") + + +@pytest.fixture +def mock_project_path(mocker): + mock_path = "/tmp/project_path" + mocker.patch("pathlib.Path.cwd", return_value=mock_path) + return mock_path + + +class TestCliRunViz: + @pytest.mark.parametrize( + "command_options,run_server_args", + [ + ( + ["viz"], + { + "host": "127.0.0.1", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {}, + "is_lite": False, + }, + ), + ( + ["viz", "run"], + { + "host": "127.0.0.1", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {}, + "is_lite": False, + }, + ), + ( + [ + "viz", + "run", + "--host", + "localhost", + ], + { + "host": "localhost", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {}, + "is_lite": False, + }, + ), + ( + [ + "viz", + "run", + "--host", + "8.8.8.8", + "--port", + "4142", + "--no-browser", + "--save-file", + "save_dir", + "--pipeline", + "data_science", + "--env", + "local", + "--params", + "extra_param=param", + ], + { + "host": "8.8.8.8", + "port": 4142, + "load_file": None, + "save_file": "save_dir", + "pipeline_name": "data_science", + "env": "local", + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {"extra_param": "param"}, + "is_lite": False, + }, + ), + ( + [ + "viz", + "run", + "--host", + "8.8.8.8", + "--port", + "4142", + "--no-browser", + "--save-file", + "save_dir", + "-p", + "data_science", + "-e", + "local", + "--params", + "extra_param=param", + ], + { + "host": "8.8.8.8", + "port": 4142, + "load_file": None, + "save_file": "save_dir", + "pipeline_name": "data_science", + "env": "local", + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {"extra_param": "param"}, + "is_lite": False, + }, + ), + ( + ["viz", "run", "--include-hooks"], + { + "host": "127.0.0.1", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "project_path": "testPath", + "autoreload": False, + "include_hooks": True, + "package_name": None, + "extra_params": {}, + "is_lite": False, + }, + ), + ( + ["viz", "run", "--lite"], + { + "host": "127.0.0.1", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "project_path": "testPath", + "autoreload": False, + "include_hooks": False, + "package_name": None, + "extra_params": {}, + "is_lite": True, + }, + ), + ], + ) + def test_kedro_viz_command_run_server( + self, + command_options, + run_server_args, + mocker, + patched_check_viz_up, + patched_start_browser, + ): + process_init = mocker.patch("multiprocessing.Process") + runner = CliRunner() + + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=run_server_args["project_path"], + ) + + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, command_options) + + process_init.assert_called_once_with( + target=run_server, daemon=False, kwargs={**run_server_args} + ) + + assert run_server_args["port"] in _VIZ_PROCESSES + + def test_kedro_viz_command_should_log_project_not_found( + self, mocker, mock_project_path, mock_click_echo + ): + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch("kedro_viz.launchers.utils._find_kedro_project", return_value=None) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [ + call( + "\x1b[31mERROR: Failed to start Kedro-Viz : " + "Could not find the project configuration " + f"file '{_PYPROJECT}' at '{mock_project_path}'. \x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_should_log_outdated_version( + self, mocker, mock_http_response, mock_click_echo, mock_project_path + ): + installed_version = parse(__version__) + mock_version = f"{installed_version.major + 1}.0.0" + requests_get = mocker.patch("requests.get") + requests_get.return_value = mock_http_response( + data={"info": {"version": mock_version}} + ) + + mocker.patch("kedro_viz.server.run_server") + + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=mock_project_path, + ) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [ + call( + "\x1b[33mWARNING: You are using an old version of Kedro Viz. " + f"You are using version {installed_version}; " + f"however, version {mock_version} is now available.\n" + "You should consider upgrading via the `pip install -U kedro-viz` command.\n" + "You can view the complete changelog at " + "https://github.com/kedro-org/kedro-viz/releases.\x1b[0m" + ) + ] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_should_not_log_latest_version( + self, mocker, mock_http_response, mock_click_echo, mock_project_path + ): + requests_get = mocker.patch("requests.get") + requests_get.return_value = mock_http_response( + data={"info": {"version": str(parse(__version__))}} + ) + + mocker.patch("kedro_viz.server.run_server") + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=mock_project_path, + ) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [call("\x1b[32mStarting Kedro Viz ...\x1b[0m")] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_should_not_log_if_pypi_is_down( + self, mocker, mock_click_echo, mock_project_path + ): + requests_get = mocker.patch("requests.get") + requests_get.side_effect = requests.exceptions.RequestException("PyPI is down") + + mocker.patch("kedro_viz.server.run_server") + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=mock_project_path, + ) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run"]) + + mock_click_echo_calls = [call("\x1b[32mStarting Kedro Viz ...\x1b[0m")] + + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + def test_kedro_viz_command_with_autoreload( + self, mocker, mock_project_path, patched_check_viz_up, patched_start_browser + ): + process_init = mocker.patch("multiprocessing.Process") + + # Reduce the timeout argument from 600 to 1 to make test run faster. + mocker.patch( + "kedro_viz.launchers.utils._wait_for.__defaults__", (True, 1, True, 1) + ) + # Mock finding kedro project + mocker.patch( + "kedro_viz.launchers.utils._find_kedro_project", + return_value=mock_project_path, + ) + runner = CliRunner() + with runner.isolated_filesystem(): + runner.invoke(main.viz_cli, ["viz", "run", "--autoreload"]) + + run_process_kwargs = { + "path": mock_project_path, + "target": run_server, + "kwargs": { + "host": "127.0.0.1", + "port": 4141, + "load_file": None, + "save_file": None, + "pipeline_name": None, + "env": None, + "autoreload": True, + "project_path": mock_project_path, + "include_hooks": False, + "package_name": None, + "extra_params": {}, + "is_lite": False, + }, + "watcher_cls": RegExpWatcher, + "watcher_kwargs": {"re_files": "^.*(\\.yml|\\.yaml|\\.py|\\.json)$"}, + } + + process_init.assert_called_once_with( + target=run_process, daemon=False, kwargs={**run_process_kwargs} + ) + assert run_process_kwargs["kwargs"]["port"] in _VIZ_PROCESSES diff --git a/package/tests/test_launchers/test_cli/test_utils.py b/package/tests/test_launchers/test_cli/test_utils.py new file mode 100644 index 0000000000..d8277c70ee --- /dev/null +++ b/package/tests/test_launchers/test_cli/test_utils.py @@ -0,0 +1,265 @@ +from unittest.mock import Mock, call, patch + +import pytest + +from kedro_viz import __version__ +from kedro_viz.constants import VIZ_DEPLOY_TIME_LIMIT +from kedro_viz.launchers.cli.utils import ( + _load_and_deploy_viz, + _viz_deploy_progress_timer, + create_shareableviz_process, +) + + +@pytest.fixture +def mock_viz_deploy_process(mocker): + return mocker.patch("multiprocessing.Process") + + +@pytest.fixture +def mock_process_completed(mocker): + return mocker.patch("multiprocessing.Value", return_value=Mock()) + + +@pytest.fixture +def mock_exception_queue(mocker): + return mocker.patch("multiprocessing.Queue", return_value=Mock()) + + +@pytest.fixture +def mock_viz_load_and_deploy(mocker): + return mocker.patch("kedro_viz.launchers.cli.utils._load_and_deploy_viz") + + +@pytest.fixture +def mock_viz_deploy_progress_timer(mocker): + return mocker.patch("kedro_viz.launchers.cli.utils._viz_deploy_progress_timer") + + +@pytest.fixture +def mock_DeployerFactory(mocker): + return mocker.patch( + "kedro_viz.integrations.deployment.deployer_factory.DeployerFactory" + ) + + +@pytest.fixture +def mock_load_and_populate_data(mocker): + return mocker.patch("kedro_viz.server.load_and_populate_data") + + +@pytest.fixture +def mock_click_echo(mocker): + return mocker.patch("click.echo") + + +@pytest.fixture +def mock_project_path(mocker): + mock_path = "/tmp/project_path" + mocker.patch("pathlib.Path.cwd", return_value=mock_path) + return mock_path + + +class TestCliUtils: + @pytest.mark.parametrize( + "platform, is_all_previews_enabled, endpoint, bucket_name," + "include_hooks, process_completed_value", + [ + ( + "azure", + True, + "https://example-bucket.web.core.windows.net", + "example-bucket", + True, + 1, + ), + ( + "aws", + True, + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "example-bucket", + True, + 1, + ), + ( + "gcp", + False, + "http://34.120.87.227/", + "example-bucket", + False, + 1, + ), + ("local", False, None, None, False, 1), + ( + "azure", + True, + "https://example-bucket.web.core.windows.net", + "example-bucket", + False, + 0, + ), + ( + "aws", + False, + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "example-bucket", + False, + 0, + ), + ( + "gcp", + True, + "http://34.120.87.227/", + "example-bucket", + True, + 0, + ), + ("local", True, None, None, True, 0), + ], + ) + def test_create_shareableviz_process( + self, + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + process_completed_value, + mock_viz_deploy_process, + mock_process_completed, + mock_exception_queue, + mock_viz_load_and_deploy, + mock_viz_deploy_progress_timer, + mock_click_echo, + ): + mock_process_completed.return_value.value = process_completed_value + create_shareableviz_process( + platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks + ) + + # Assert the mocks were called as expected + mock_viz_deploy_process.assert_called_once_with( + target=mock_viz_load_and_deploy, + args=( + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + None, + mock_process_completed.return_value, + mock_exception_queue.return_value, + ), + ) + mock_viz_deploy_process.return_value.start.assert_called_once() + mock_viz_deploy_progress_timer.assert_called_once_with( + mock_process_completed.return_value, VIZ_DEPLOY_TIME_LIMIT + ) + mock_viz_deploy_process.return_value.terminate.assert_called_once() + + if process_completed_value: + if platform != "local": + msg = ( + "\x1b[32m\u2728 Success! Kedro Viz has been deployed on " + f"{platform.upper()}. " + "It can be accessed at :\n" + f"{endpoint}\x1b[0m" + ) + else: + msg = ( + "\x1b[32m✨ Success! Kedro-Viz build files have been " + "added to the `build` directory.\x1b[0m" + ) + else: + msg = ( + "\x1b[31mTIMEOUT ERROR: Failed to build/deploy Kedro-Viz " + f"as the process took more than {VIZ_DEPLOY_TIME_LIMIT} seconds. " + "Please try again later.\x1b[0m" + ) + + mock_click_echo_calls = [call(msg)] + mock_click_echo.assert_has_calls(mock_click_echo_calls) + + @pytest.mark.parametrize( + "platform, is_all_previews_enabled, endpoint, bucket_name, include_hooks, package_name", + [ + ( + "azure", + False, + "https://example-bucket.web.core.windows.net", + "example-bucket", + False, + "demo_project", + ), + ( + "aws", + True, + "http://example-bucket.s3-website.us-east-2.amazonaws.com/", + "example-bucket", + True, + "demo_project", + ), + ( + "gcp", + True, + "http://34.120.87.227/", + "example-bucket", + False, + "demo_project", + ), + ("local", False, None, None, True, "demo_project"), + ], + ) + def test_load_and_deploy_viz_success( + self, + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + package_name, + mock_DeployerFactory, + mock_load_and_populate_data, + mock_process_completed, + mock_exception_queue, + mock_click_echo, + mock_project_path, + ): + deployer_mock = mock_DeployerFactory.create_deployer.return_value + + _load_and_deploy_viz( + platform, + is_all_previews_enabled, + endpoint, + bucket_name, + include_hooks, + package_name, + mock_process_completed, + mock_exception_queue, + ) + + mock_load_and_populate_data.assert_called_once_with( + mock_project_path, include_hooks=include_hooks, package_name=package_name + ) + mock_DeployerFactory.create_deployer.assert_called_once_with( + platform, endpoint, bucket_name + ) + deployer_mock.deploy.assert_called_once_with(is_all_previews_enabled) + mock_click_echo.echo.assert_not_called() + + def test_viz_deploy_progress_timer(self, capsys): + mock_process_completed = Mock() + mock_process_completed.value = 0 + + with patch("kedro_viz.launchers.cli.utils.sleep") as mock_sleep: + _viz_deploy_progress_timer(mock_process_completed, VIZ_DEPLOY_TIME_LIMIT) + + assert mock_sleep.call_count == VIZ_DEPLOY_TIME_LIMIT + 1 + + expected_sleep_calls = [call(1)] * (VIZ_DEPLOY_TIME_LIMIT + 1) + mock_sleep.assert_has_calls(expected_sleep_calls) + captured = capsys.readouterr() + + for second in range(1, VIZ_DEPLOY_TIME_LIMIT + 1): + expected_output = f"...Creating your build/deploy Kedro-Viz ({second}s)" + assert expected_output in captured.out diff --git a/package/tests/test_launchers/test_utils.py b/package/tests/test_launchers/test_utils.py index 04425cfd09..83e9203bd3 100644 --- a/package/tests/test_launchers/test_utils.py +++ b/package/tests/test_launchers/test_utils.py @@ -1,17 +1,15 @@ from pathlib import Path from unittest import mock -from unittest.mock import Mock, call, patch +from unittest.mock import Mock import pytest import requests -from kedro_viz.constants import VIZ_DEPLOY_TIME_LIMIT from kedro_viz.launchers.utils import ( _check_viz_up, _find_kedro_project, _is_project, _start_browser, - viz_deploy_progress_timer, ) @@ -56,24 +54,6 @@ def test_check_viz_up(host, port, status_code, expected_result, mocker): assert result == expected_result -def test_viz_deploy_progress_timer(capsys): - mock_process_completed = Mock() - mock_process_completed.value = 0 - - with patch("kedro_viz.launchers.utils.sleep") as mock_sleep: - viz_deploy_progress_timer(mock_process_completed, VIZ_DEPLOY_TIME_LIMIT) - - assert mock_sleep.call_count == VIZ_DEPLOY_TIME_LIMIT + 1 - - expected_sleep_calls = [call(1)] * (VIZ_DEPLOY_TIME_LIMIT + 1) - mock_sleep.assert_has_calls(expected_sleep_calls) - captured = capsys.readouterr() - - for second in range(1, VIZ_DEPLOY_TIME_LIMIT + 1): - expected_output = f"...Creating your build/deploy Kedro-Viz ({second}s)" - assert expected_output in captured.out - - class TestIsProject: project_path = Path.cwd() diff --git a/package/tests/test_models/test_flowchart.py b/package/tests/test_models/test_flowchart.py index 587cfb49af..521cc419f8 100644 --- a/package/tests/test_models/test_flowchart.py +++ b/package/tests/test_models/test_flowchart.py @@ -315,7 +315,7 @@ def identity(x): assert not task_node_metadata.parameters assert ( task_node_metadata.run_command - == "kedro run --to-nodes=namespace.identity_node" + == "kedro run --to-nodes='namespace.identity_node'" ) def test_task_node_metadata_no_namespace(self): @@ -338,9 +338,9 @@ def identity(x): Path(__file__).relative_to(Path.cwd().parent).expanduser() ) assert not task_node_metadata.parameters - assert task_node_metadata.run_command == "kedro run --to-nodes=identity_node" + assert task_node_metadata.run_command == "kedro run --to-nodes='identity_node'" - def test_task_node_metadata_no_run_command(self): + def test_task_node_metadata_no_name(self): kedro_node = node( identity, inputs="x", @@ -352,7 +352,10 @@ def test_task_node_metadata_no_run_command(self): kedro_node, "identity_node", set(["namespace"]) ) task_node_metadata = TaskNodeMetadata(task_node=task_node) - assert task_node_metadata.run_command is None + assert ( + task_node_metadata.run_command + == f"kedro run --to-nodes='{kedro_node.name}'" + ) def test_task_node_metadata_with_decorated_func(self): kedro_node = node( diff --git a/package/tests/test_models/test_metadata.py b/package/tests/test_models/test_metadata.py new file mode 100644 index 0000000000..d81bb04502 --- /dev/null +++ b/package/tests/test_models/test_metadata.py @@ -0,0 +1,70 @@ +import pytest +from pydantic import ValidationError + +from kedro_viz.models.metadata import Metadata, PackageCompatibility + + +class TestPackageCompatibility: + def test_package_compatibility_valid_data(self): + package = PackageCompatibility( + package_name="kedro", package_version="0.18.0", is_compatible=True + ) + assert package.package_name == "kedro" + assert package.package_version == "0.18.0" + assert package.is_compatible is True + + def test_package_compatibility_invalid_package_name(self): + with pytest.raises(ValidationError) as excinfo: + PackageCompatibility( + package_name=123, # invalid type + package_version="0.18.0", + is_compatible=True, + ) + assert "Input should be a valid string" in str(excinfo.value) + + def test_package_compatibility_invalid_package_version(self): + with pytest.raises(ValidationError) as excinfo: + PackageCompatibility( + package_name="kedro", + package_version=123, # invalid type + is_compatible=True, + ) + assert "Input should be a valid string" in str(excinfo.value) + + def test_package_compatibility_invalid_is_compatible(self): + with pytest.raises(ValidationError) as excinfo: + PackageCompatibility( + package_name="kedro", + package_version="0.18.0", + is_compatible="random", # invalid type + ) + assert "Input should be a valid boolean" in str(excinfo.value) + + +class TestMetadata: + def test_metadata_default_values(self): + # Test default values of Metadata + assert Metadata.has_missing_dependencies is False + assert not Metadata.package_compatibilities + + def test_metadata_set_package_compatibilities(self): + kedro_package = PackageCompatibility( + package_name="kedro", package_version="0.18.0", is_compatible=True + ) + pandas_package = PackageCompatibility( + package_name="pandas", package_version="1.2.0", is_compatible=False + ) + + # Set the package compatibilities using the class method + Metadata.set_package_compatibilities([kedro_package, pandas_package]) + + # Assert the values have been set correctly + assert Metadata.package_compatibilities == [kedro_package, pandas_package] + + def test_metadata_set_has_missing_dependencies(self): + # Test changing the has_missing_dependencies value + Metadata.set_has_missing_dependencies(True) + assert Metadata.has_missing_dependencies is True + + Metadata.set_has_missing_dependencies(False) + assert Metadata.has_missing_dependencies is False diff --git a/src/actions/index.js b/src/actions/index.js index a04c759fef..e169c38900 100644 --- a/src/actions/index.js +++ b/src/actions/index.js @@ -141,6 +141,15 @@ export function toggleShowFeatureHints(showFeatureHints) { }; } +export const TOGGLE_SHOW_DATASET_PREVIEWS = 'TOGGLE_SHOW_DATASET_PREVIEWS'; + +export function toggleShowDatasetPreviews(showDatasetPreviews) { + return { + type: TOGGLE_SHOW_DATASET_PREVIEWS, + showDatasetPreviews, + }; +} + export const TOGGLE_SIDEBAR = 'TOGGLE_SIDEBAR'; /** @@ -221,6 +230,21 @@ export function changeFlag(name, value) { }; } +export const SET_BANNER = 'SET_BANNER'; + +/** + * Change the given banner status + * @param {String} name The banner name + * @param {Value} value The value to set + */ +export function setBanner(name, value) { + return { + type: SET_BANNER, + name, + value, + }; +} + export const TOGGLE_IGNORE_LARGE_WARNING = 'TOGGLE_IGNORE_LARGE_WARNING'; /** diff --git a/src/actions/pipelines.js b/src/actions/pipelines.js index aabdf139a8..05a9c23539 100644 --- a/src/actions/pipelines.js +++ b/src/actions/pipelines.js @@ -1,6 +1,9 @@ import { getUrl } from '../utils'; import loadJsonData from '../store/load-data'; -import { preparePipelineState } from '../store/initial-state'; +import { + parseUrlParameters, + preparePipelineState, +} from '../store/initial-state'; import { resetData } from './index'; /** @@ -99,14 +102,15 @@ export function loadInitialPipelineData() { // obtain the status of expandAllPipelines to decide whether it needs to overwrite the // list of visible nodes const expandAllPipelines = state.expandAllPipelines; + const urlParams = parseUrlParameters(); let newState = await loadJsonData(url).then((data) => - preparePipelineState(data, true, expandAllPipelines) + preparePipelineState(data, true, expandAllPipelines, urlParams) ); // If the active pipeline isn't 'main' then request data from new URL if (requiresSecondRequest(newState.pipeline)) { const url = getPipelineUrl(newState.pipeline); newState = await loadJsonData(url).then((data) => - preparePipelineState(data, false, expandAllPipelines) + preparePipelineState(data, false, expandAllPipelines, urlParams) ); } dispatch(resetData(newState)); diff --git a/src/actions/preferences.js b/src/actions/preferences.js deleted file mode 100644 index 2dade70770..0000000000 --- a/src/actions/preferences.js +++ /dev/null @@ -1,19 +0,0 @@ -import { fetchPreferences } from '../utils/preferences-api'; - -// Action Types -export const UPDATE_USER_PREFERENCES = 'UPDATE_USER_PREFERENCES'; - -// Action Creators -export const updateUserPreferences = (preferences) => ({ - type: UPDATE_USER_PREFERENCES, - payload: preferences, -}); - -export const getPreferences = () => async (dispatch) => { - try { - const preferences = await fetchPreferences(); - dispatch(updateUserPreferences(preferences)); - } catch (error) { - console.error('Error fetching preferences:', error); - } -}; diff --git a/src/actions/slice.js b/src/actions/slice.js new file mode 100644 index 0000000000..12fee72784 --- /dev/null +++ b/src/actions/slice.js @@ -0,0 +1,28 @@ +export const APPLY_SLICE_PIPELINE = 'APPLY_SLICE_PIPELINE'; + +export const applySlicePipeline = (apply) => { + return async function (dispatch) { + dispatch({ + type: APPLY_SLICE_PIPELINE, + apply, + }); + }; +}; + +export const SET_SLICE_PIPELINE = 'SET_SLICE_PIPELINE'; + +export const setSlicePipeline = (from, to) => { + return async function (dispatch) { + dispatch({ + type: SET_SLICE_PIPELINE, + slice: { from, to }, + }); + }; +}; + +export const RESET_SLICE_PIPELINE = 'RESET_SLICE_PIPELINE'; + +export const resetSlicePipeline = () => ({ + type: RESET_SLICE_PIPELINE, + slice: { from: null, to: null }, +}); diff --git a/src/components/app/app.js b/src/components/app/app.js index 2eb3a2abe6..340dc1e44e 100644 --- a/src/components/app/app.js +++ b/src/components/app/app.js @@ -25,7 +25,11 @@ class App extends React.Component { constructor(props) { super(props); const initialState = getInitialState(props); - this.store = configureStore(initialState, this.props.data); + this.store = configureStore( + initialState, + this.props.data, + this.props.onActionCallback + ); } componentDidMount() { diff --git a/src/components/app/app.scss b/src/components/app/app.scss index bb1a3ea9a1..bd53527605 100644 --- a/src/components/app/app.scss +++ b/src/components/app/app.scss @@ -10,7 +10,7 @@ --color-bg-1: #{colors.$white-800}; --color-bg-2: #{colors.$grey-0}; --color-bg-3: #{colors.$white-200}; - --color-bg-4: #{colors.$white-0}; + --color-bg-4: #{colors.$white-200}; --color-bg-5: #{colors.$white-600}; --color-bg-alt: #{colors.$black-700}; --color-bg-list: #{colors.$white-100}; diff --git a/src/components/app/app.test.js b/src/components/app/app.test.js index 33d0e33c79..b56b429814 100644 --- a/src/components/app/app.test.js +++ b/src/components/app/app.test.js @@ -12,7 +12,6 @@ import { localStorageName } from '../../config'; import { prepareNonPipelineState } from '../../store/initial-state'; import reducer from '../../reducers/index'; import { TOGGLE_GRAPH_LOADING } from '../../actions/graph'; -import { prettifyName } from '../../utils/index'; describe('App', () => { const getState = (wrapper) => wrapper.instance().store.getState(); diff --git a/src/components/experiment-wrapper/experiment-wrapper.js b/src/components/experiment-wrapper/experiment-wrapper.js index ce9fb3b718..ce1195e7c1 100644 --- a/src/components/experiment-wrapper/experiment-wrapper.js +++ b/src/components/experiment-wrapper/experiment-wrapper.js @@ -18,7 +18,7 @@ import { PACKAGE_KEDRO_DATASETS, } from '../../config'; import { findMatchedPath } from '../../utils/match-path'; -import { fetchPackageCompatibilities } from '../../utils'; +import { fetchMetadata } from '../../utils'; import { saveLocalStorage, loadLocalStorage } from '../../store/helpers'; import './experiment-wrapper.scss'; @@ -193,23 +193,24 @@ const ExperimentWrapper = ({ theme, runsMetadata }) => { }, []); useEffect(() => { - async function fetchPackageCompatibility() { + async function checkPackageCompatibility() { try { - const request = await fetchPackageCompatibilities(); + const request = await fetchMetadata(); const response = await request.json(); if (request.ok) { - const kedroDatasetsPackage = response.find( + const packageCompatibilityInfo = response.package_compatibilities; + const kedroDatasetsPackage = packageCompatibilityInfo.find( (pckg) => pckg.package_name === PACKAGE_KEDRO_DATASETS ); setIsKedroDatasetsCompatible(kedroDatasetsPackage.is_compatible); } } catch (error) { - console.error('package-compatibilities fetch error: ', error); + console.error('metadata fetch error: ', error); } } - fetchPackageCompatibility(); + checkPackageCompatibility(); }, []); useEffect(() => { diff --git a/src/components/feedback-button/feedback-button.js b/src/components/feedback-button/feedback-button.js new file mode 100644 index 0000000000..eca5241ad1 --- /dev/null +++ b/src/components/feedback-button/feedback-button.js @@ -0,0 +1,17 @@ +import React from 'react'; +import classnames from 'classnames'; + +import './feedback-button.scss'; + +export const FeedbackButton = ({ onClick, visible, title }) => { + return ( + + ); +}; diff --git a/src/components/feedback-button/feedback-button.scss b/src/components/feedback-button/feedback-button.scss new file mode 100755 index 0000000000..01f3c6f006 --- /dev/null +++ b/src/components/feedback-button/feedback-button.scss @@ -0,0 +1,21 @@ +.feedback-button { + background-color: #007bff; + border: none; + color: white; + cursor: pointer; + font-size: 16px; + height: 40px; + line-height: 40px; + opacity: 0; + position: fixed; + right: -96px; + text-align: center; + top: 50%; + padding: 0 16px; + transform: translateY(-50%) rotate(90deg); +} + +.feedback-button--visible { + opacity: 1; + transition: opacity 0.5s ease-in; +} diff --git a/src/components/feedback-form/feedback-form.js b/src/components/feedback-form/feedback-form.js new file mode 100755 index 0000000000..a7f32b66f3 --- /dev/null +++ b/src/components/feedback-form/feedback-form.js @@ -0,0 +1,110 @@ +import React, { useState, useEffect } from 'react'; +import classnames from 'classnames'; +import Button from '../ui/button'; +import CloseIcon from '../icons/close'; +import { Mood } from '../mood/mood'; +import { getHeap } from '../../tracking'; +import { getDataTestAttribute } from '../../utils/get-data-test-attribute'; +import { loadLocalStorage, saveLocalStorage } from '../../store/helpers'; +import { localStorageFeedbackSeen } from '../../config'; + +import './feedback-form.scss'; + +export const FeedbackForm = ({ hideForm, title, usageContext }) => { + const [formStatus, setFormStatus] = useState('active'); // 'active', 'submitted', 'cancelled' + const [activeMood, setActiveMood] = useState(null); + const [feedbackText, setFeedbackText] = useState(''); + + const handleFormAction = (action) => { + setFormStatus(action); + + const timer = setTimeout(() => { + updateLocalStorageUsageContext(false); + hideForm(); + }, 4000); + + return () => clearTimeout(timer); + }; + + const updateLocalStorageUsageContext = (value) => { + const existingData = loadLocalStorage(localStorageFeedbackSeen) || {}; + existingData[usageContext] = value; + saveLocalStorage(localStorageFeedbackSeen, existingData); + }; + + useEffect(() => { + if (formStatus === 'submitted' || formStatus === 'cancelled') { + const timer = setTimeout(() => { + setFormStatus('active'); + setActiveMood(null); + setFeedbackText(''); + }, 4000); + + return () => clearTimeout(timer); + } + }, [formStatus]); + + const handleFormSubmit = (e) => { + e.preventDefault(); + handleFormAction('submitted'); + getHeap().track(getDataTestAttribute(usageContext, 'feedback-form'), { + rating: activeMood, + feedback: feedbackText, + }); + }; + + const getMessages = () => { + if (formStatus === 'submitted') { + return 'Thank you for sharing feedback!'; + } + if (formStatus === 'cancelled') { + return ( + <> + You can provide feedback any time by using +
+ the feedback button in the sliced view. + + ); + } + }; + + if (formStatus === 'submitted' || formStatus === 'cancelled') { + return ( +
+ {getMessages()} +
+ ); + } else { + return ( +
+
handleFormAction('cancelled')} + > + +
+

{title}

+
+ + {activeMood !== null && ( + <> +