From 9d0f1787417e0c696407db15692a84ffa96645cc Mon Sep 17 00:00:00 2001 From: Paul Abumov Date: Thu, 1 Aug 2024 18:32:58 -0400 Subject: [PATCH] Updated NodeJS to v22 --- .../docker-testing-matrix-manual-install.yml | 2 +- Dockerfile | 2 +- docker/dockerfiles/Dockerfile.ubuntu-24.04 | 4 +- .../manual_installation.md | 2 +- examples/form_composer_demo/run_task.py | 50 +-- .../form_composer_demo/run_task_dynamic.py | 72 +--- .../run_task_dynamic_ec2_mturk_sandbox.py | 103 ++---- .../run_task_dynamic_ec2_prolific.py | 105 ++---- ...ask_dynamic_presigned_urls_ec2_prolific.py | 93 ++--- .../run_task_with_gold_unit.py | 48 +-- .../run_task_with_onboarding.py | 56 +-- .../run_task_with_screening.py | 52 +-- .../run_task_with_worker_opinion.py | 57 +-- examples/parlai_chat_task_demo/run_task.py | 20 +- examples/remote_procedure/mnist/model.py | 24 +- examples/remote_procedure/mnist/run_task.py | 98 ++--- .../mnist/webapp/package.json | 2 +- .../remote_procedure/mnist/webapp/src/app.jsx | 2 +- .../mnist/webapp/webpack.config.js | 5 + .../mnist/webapp/webpack.config.review.js | 5 + .../remote_procedure/template/run_task.py | 70 ++-- .../template/webapp/package.json | 2 +- .../template/webapp/src/app.jsx | 2 +- .../template/webapp/webpack.config.js | 5 + .../toxicity_detection/run_task.py | 59 +-- .../toxicity_detection/webapp/package.json | 2 +- .../toxicity_detection/webapp/src/app.jsx | 2 +- .../webapp/webpack.config.js | 5 + examples/static_react_task/run_task.py | 32 +- .../run_task.py | 46 +-- mephisto/client/cli_review_app_commands.py | 11 +- mephisto/client/cli_scripts_commands.py | 19 +- mephisto/generators/form_composer/run.py | 52 +-- .../scripts/form_composer/rebuild_all_apps.py | 246 ++----------- mephisto/scripts/tests/__init__.py | 7 + mephisto/scripts/tests/react_apps.py | 127 +++++++ .../tools/building_react_apps/__init__.py | 5 + .../tools/building_react_apps/examples.py | 340 ++++++++++++++++++ .../tools/building_react_apps/generators.py | 82 +++++ .../tools/building_react_apps/packages.py | 246 +++++++++++++ .../tools/building_react_apps/review_app.py | 41 +++ mephisto/tools/building_react_apps/utils.py | 31 ++ 42 files changed, 1275 insertions(+), 959 deletions(-) create mode 100644 mephisto/scripts/tests/__init__.py create mode 100644 mephisto/scripts/tests/react_apps.py create mode 100644 mephisto/tools/building_react_apps/__init__.py create mode 100644 mephisto/tools/building_react_apps/examples.py create mode 100644 mephisto/tools/building_react_apps/generators.py create mode 100644 mephisto/tools/building_react_apps/packages.py create mode 100644 mephisto/tools/building_react_apps/review_app.py create mode 100644 mephisto/tools/building_react_apps/utils.py diff --git a/.github/workflows/docker-testing-matrix-manual-install.yml b/.github/workflows/docker-testing-matrix-manual-install.yml index 31fca7abe..2598950f5 100644 --- a/.github/workflows/docker-testing-matrix-manual-install.yml +++ b/.github/workflows/docker-testing-matrix-manual-install.yml @@ -43,4 +43,4 @@ jobs: - name: Run command that removes and rebuilds all React apps related to the FormComposer run: | - docker run mephisto_manual bash -c 'mephisto scripts form_composer rebuild_all_apps' + docker run mephisto_manual bash -c 'mephisto scripts tests rebuild_all_mephisto_react_apps' diff --git a/Dockerfile b/Dockerfile index 542ff92bc..77f186e81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Using the -slim version below for minimal size. You may want to # remove -slim, or switch to -alpine if encountering issues -ARG BASE_TAG=python3.9-nodejs16-slim +ARG BASE_TAG=python3.9-nodejs22-slim ARG BASE_IMAGE=nikolaik/python-nodejs:$BASE_TAG FROM $BASE_IMAGE diff --git a/docker/dockerfiles/Dockerfile.ubuntu-24.04 b/docker/dockerfiles/Dockerfile.ubuntu-24.04 index 8e450765b..2ff585a22 100644 --- a/docker/dockerfiles/Dockerfile.ubuntu-24.04 +++ b/docker/dockerfiles/Dockerfile.ubuntu-24.04 @@ -26,7 +26,7 @@ RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | b RUN export NVM_DIR="$HOME/.nvm" \ && [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" \ && [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" \ - && nvm install 16 \ + && nvm install 22 \ && ln -s $(which node) /usr/bin/node \ && ln -s $(which npm) /usr/bin/npm @@ -43,6 +43,8 @@ COPY . $MEPHISTO_REPO_PATH # Upgrade pip so we can use the `pyproject.toml` without raising an error RUN pip install --upgrade pip # Install Python requirements +# [FOR DOCKERFILE ONLY] Requirements for Mephisto example `mnist`. Uncomment if you need them +# RUN pip install torch pillow numpy detoxify # [FOR DOCKERFILE ONLY] `--ignore-installed` - some libs can be preinstall in Docker-system # Use `cd /mephisto && pip install -e .` in your local environment RUN cd /mephisto && pip install --ignore-installed -e . diff --git a/docs/web/docs/guides/how_to_use/efficiency_organization/manual_installation.md b/docs/web/docs/guides/how_to_use/efficiency_organization/manual_installation.md index 2ff2e7c31..c4bfe2773 100644 --- a/docs/web/docs/guides/how_to_use/efficiency_organization/manual_installation.md +++ b/docs/web/docs/guides/how_to_use/efficiency_organization/manual_installation.md @@ -62,7 +62,7 @@ curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash export NVM_DIR="$HOME/.nvm" \ && [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" \ && [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" \ - && nvm install 16 \ + && nvm install 22 \ && ln -s $(which node) /usr/bin/node \ && ln -s $(which npm) /usr/bin/npm npm install -g yarn diff --git a/examples/form_composer_demo/run_task.py b/examples/form_composer_demo/run_task.py index a27617519..4169435e6 100644 --- a/examples/form_composer_demo/run_task.py +++ b/examples/form_composer_demo/run_task.py @@ -4,63 +4,21 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -import os - from omegaconf import DictConfig from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script @task_script(default_config_file="example_local_mock") def main(operator: Operator, cfg: DictConfig) -> None: - # Build packages - _build_custom_bundles(cfg) - - operator.launch_task_run(cfg.mephisto) - operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) - - -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:simple:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, + examples.build_form_composer_simple( force_rebuild=cfg.mephisto.task.force_rebuild, post_install_script=cfg.mephisto.task.post_install_script, - build_command="dev:simple", ) + operator.launch_task_run(cfg.mephisto) + operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) if __name__ == "__main__": diff --git a/examples/form_composer_demo/run_task_dynamic.py b/examples/form_composer_demo/run_task_dynamic.py index 06e8b0085..0baf14bc6 100644 --- a/examples/form_composer_demo/run_task_dynamic.py +++ b/examples/form_composer_demo/run_task_dynamic.py @@ -15,65 +15,12 @@ from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) -from mephisto.generators.form_composer.config_validation.utils import set_custom_triggers_js_env_var -from mephisto.generators.form_composer.config_validation.utils import ( - set_custom_validators_js_env_var, -) from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script -@task_script(default_config_file="dynamic_example_local_mock") -def main(operator: Operator, cfg: DictConfig) -> None: - # Build packages - _build_custom_bundles(cfg) - - operator.launch_task_run(cfg.mephisto) - operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) - - -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - ) - - -def generate_task_data_json_config(): +def _generate_task_data_json_config(): """ Generate extrapolated `task_data.json` config file, based on existing form and tokens values config files @@ -95,12 +42,17 @@ def generate_task_data_json_config(): data_path=data_path, ) - # Set env var for `custom_validators.js` - set_custom_validators_js_env_var(data_path) - # Set env var for `custom_triggers.js` - set_custom_triggers_js_env_var(data_path) + +@task_script(default_config_file="dynamic_example_local_mock") +def main(operator: Operator, cfg: DictConfig) -> None: + examples.build_form_composer_dynamic( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) + operator.launch_task_run(cfg.mephisto) + operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) if __name__ == "__main__": - generate_task_data_json_config() + _generate_task_data_json_config() main() diff --git a/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py b/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py index 415905ee0..edf9e92d5 100644 --- a/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py +++ b/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py @@ -21,79 +21,14 @@ from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) -from mephisto.generators.form_composer.config_validation.utils import set_custom_triggers_js_env_var -from mephisto.generators.form_composer.config_validation.utils import ( - set_custom_validators_js_env_var, -) from mephisto.generators.form_composer.constants import TOKEN_END_REGEX from mephisto.generators.form_composer.constants import TOKEN_START_REGEX from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script -@task_script(default_config_file="dynamic_example_ec2_mturk_sandbox") -def main(operator: Operator, cfg: DictConfig) -> None: - # Build packages - _build_custom_bundles(cfg) - - shared_state = SharedStaticTaskState() - - # Mephisto qualifications - shared_state.qualifications = [ - # Custom Mephisto qualifications - ] - - # Mturk qualifications - shared_state.mturk_specific_qualifications = [ - # MTurk-specific quality control qualifications - ] - - operator.launch_task_run(cfg.mephisto, shared_state) - operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) - - -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - ) - - -def generate_data_json_config(): +def _generate_data_json_config(): """ Generate extrapolated `task_data.json` config file, based on existing form and tokens values config files @@ -115,13 +50,8 @@ def generate_data_json_config(): data_path=data_path, ) - # Set env var for `custom_validators.js` - set_custom_validators_js_env_var(data_path) - # Set env var for `custom_triggers.js` - set_custom_triggers_js_env_var(data_path) - -def generate_preview_html(): +def _generate_preview_html(): """ Generate HTML preview of a Task (based on first form version contained in `task_data.json`) """ @@ -168,7 +98,30 @@ def generate_preview_html(): f.write(preview_template) +@task_script(default_config_file="dynamic_example_ec2_mturk_sandbox") +def main(operator: Operator, cfg: DictConfig) -> None: + examples.build_form_composer_dynamic_ec2_mturk_sandbox( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) + + shared_state = SharedStaticTaskState() + + # Mephisto qualifications + shared_state.qualifications = [ + # Custom Mephisto qualifications + ] + + # Mturk qualifications + shared_state.mturk_specific_qualifications = [ + # MTurk-specific quality control qualifications + ] + + operator.launch_task_run(cfg.mephisto, shared_state) + operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) + + if __name__ == "__main__": - generate_data_json_config() - generate_preview_html() + _generate_data_json_config() + _generate_preview_html() main() diff --git a/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py b/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py index cc99edbe7..634af0e81 100644 --- a/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py +++ b/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py @@ -19,20 +19,41 @@ from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) -from mephisto.generators.form_composer.config_validation.utils import set_custom_triggers_js_env_var -from mephisto.generators.form_composer.config_validation.utils import ( - set_custom_validators_js_env_var, -) from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script from mephisto.utils.qualifications import make_qualification_dict +def _generate_data_json_config(): + """ + Generate extrapolated `task_data.json` config file, + based on existing form and tokens values config files + """ + app_path = os.path.dirname(os.path.abspath(__file__)) + data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME, "dynamic") + + form_config_path = os.path.join(data_path, FORM_COMPOSER__FORM_CONFIG_NAME) + token_sets_values_config_path = os.path.join( + data_path, + FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME, + ) + task_data_config_path = os.path.join(data_path, FORM_COMPOSER__DATA_CONFIG_NAME) + + create_extrapolated_config( + form_config_path=form_config_path, + token_sets_values_config_path=token_sets_values_config_path, + task_data_config_path=task_data_config_path, + data_path=data_path, + ) + + @task_script(default_config_file="dynamic_example_ec2_prolific") def main(operator: Operator, cfg: DictConfig) -> None: - # Build packages - _build_custom_bundles(cfg) + examples.build_form_composer_dynamic_ec2_prolific( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) shared_state = SharedStaticTaskState() @@ -55,74 +76,6 @@ def main(operator: Operator, cfg: DictConfig) -> None: operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - ) - - -def generate_data_json_config(): - """ - Generate extrapolated `task_data.json` config file, - based on existing form and tokens values config files - """ - app_path = os.path.dirname(os.path.abspath(__file__)) - data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME, "dynamic") - - form_config_path = os.path.join(data_path, FORM_COMPOSER__FORM_CONFIG_NAME) - token_sets_values_config_path = os.path.join( - data_path, - FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME, - ) - task_data_config_path = os.path.join(data_path, FORM_COMPOSER__DATA_CONFIG_NAME) - - create_extrapolated_config( - form_config_path=form_config_path, - token_sets_values_config_path=token_sets_values_config_path, - task_data_config_path=task_data_config_path, - data_path=data_path, - ) - - # Set env var for `custom_validators.js` - set_custom_validators_js_env_var(data_path) - # Set env var for `custom_triggers.js` - set_custom_triggers_js_env_var(data_path) - - if __name__ == "__main__": - generate_data_json_config() + _generate_data_json_config() main() diff --git a/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py b/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py index 6595c7ec9..2c4aee6df 100644 --- a/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py +++ b/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py @@ -21,74 +21,11 @@ from mephisto.generators.form_composer.config_validation.utils import read_config_file from mephisto.generators.form_composer.remote_procedures import JS_NAME_FUNCTION_MAPPING from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script -@task_script(default_config_file="dynamic_presigned_urls_example_ec2_prolific") -def main(operator: Operator, cfg: DictConfig) -> None: - # Build packages - _build_custom_bundles(cfg) - - # Configure shared state - task_data_config_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "data", - "dynamic_presigned_urls", - "task_data.json", - ) - task_data = read_config_file(task_data_config_path) - shared_state = SharedRemoteProcedureTaskState( - static_task_data=task_data, - function_registry=JS_NAME_FUNCTION_MAPPING, - ) - - operator.launch_task_run(cfg.mephisto, shared_state) - operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) - - -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - build_command="build:presigned_urls", - ) - - -def generate_data_json_config(): +def _generate_data_json_config(): """ Generate extrapolated `task_data.json` config file, based on existing form and tokens values config files @@ -117,6 +54,30 @@ def generate_data_json_config(): ) +@task_script(default_config_file="dynamic_presigned_urls_example_ec2_prolific") +def main(operator: Operator, cfg: DictConfig) -> None: + examples.build_form_composer_dynamic_presigned_urls_ec2_prolific( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) + + # Configure shared state + task_data_config_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data", + "dynamic_presigned_urls", + "task_data.json", + ) + task_data = read_config_file(task_data_config_path) + shared_state = SharedRemoteProcedureTaskState( + static_task_data=task_data, + function_registry=JS_NAME_FUNCTION_MAPPING, + ) + + operator.launch_task_run(cfg.mephisto, shared_state) + operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) + + if __name__ == "__main__": - generate_data_json_config() + _generate_data_json_config() main() diff --git a/examples/form_composer_demo/run_task_with_gold_unit.py b/examples/form_composer_demo/run_task_with_gold_unit.py index 603466b53..e02cffc24 100644 --- a/examples/form_composer_demo/run_task_with_gold_unit.py +++ b/examples/form_composer_demo/run_task_with_gold_unit.py @@ -22,51 +22,10 @@ from mephisto.abstractions.blueprints.mixins.use_gold_unit import get_gold_factory from mephisto.abstractions.blueprints.mixins.use_gold_unit import UseGoldUnit from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:simple:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - build_command="dev:simple", - ) - - def _get_gold_data() -> List[Dict[str, Any]]: gold_data_path = os.path.join( # Root project directory @@ -90,7 +49,10 @@ def _get_gold_data() -> List[Dict[str, Any]]: @task_script(default_config_file="example_local_mock_with_gold_unit") def main(operator: Operator, cfg: DictConfig) -> None: # 1. Build packages - _build_custom_bundles(cfg) + examples.build_form_composer_simple_with_gold_unit( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) # 2. Prepare ShareState with Gold Units gold_data = _get_gold_data() diff --git a/examples/form_composer_demo/run_task_with_onboarding.py b/examples/form_composer_demo/run_task_with_onboarding.py index a83c54a79..0b3167518 100644 --- a/examples/form_composer_demo/run_task_with_onboarding.py +++ b/examples/form_composer_demo/run_task_with_onboarding.py @@ -4,61 +4,18 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -import os - from omegaconf import DictConfig from mephisto.abstractions.blueprints.abstract.static_task.static_blueprint import ( SharedStaticTaskState, ) from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:simple:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - build_command="dev:simple", - ) - - -def handle_onboarding(onboarding_data: dict) -> bool: - if onboarding_data["outputs"]["success"] == True: +def _handle_onboarding(onboarding_data: dict) -> bool: + if onboarding_data["outputs"]["success"] is True: return True return False @@ -67,11 +24,14 @@ def handle_onboarding(onboarding_data: dict) -> bool: @task_script(default_config_file="example_local_mock_with_oboarding") def main(operator: Operator, cfg: DictConfig) -> None: # 1. Build packages - _build_custom_bundles(cfg) + examples.build_form_composer_simple_with_onboarding( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) # 2. Prepare ShareState with Onboarding shared_state = SharedStaticTaskState( - validate_onboarding=handle_onboarding, + validate_onboarding=_handle_onboarding, ) # 3. Launch TaskRun diff --git a/examples/form_composer_demo/run_task_with_screening.py b/examples/form_composer_demo/run_task_with_screening.py index 531ec2c24..f8d3f1326 100644 --- a/examples/form_composer_demo/run_task_with_screening.py +++ b/examples/form_composer_demo/run_task_with_screening.py @@ -17,52 +17,11 @@ ) from mephisto.abstractions.blueprints.mixins.screen_task_required import ScreenTaskRequired from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:simple:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - build_command="dev:simple", - ) - - -def screening_unit_factory() -> dict: +def _screening_unit_factory() -> dict: while True: screening_data_path = os.path.join( # Root project directory @@ -90,7 +49,10 @@ def screening_unit_factory() -> dict: @task_script(default_config_file="example_local_mock_with_screening") def main(operator: Operator, cfg: DictConfig) -> None: # 1. Build packages - _build_custom_bundles(cfg) + examples.build_form_composer_simple_with_screening( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) # 2. Prepare ShareState with Screeining shared_state = SharedStaticTaskState() @@ -99,7 +61,7 @@ def main(operator: Operator, cfg: DictConfig) -> None: cfg.mephisto, validate_screening_unit, ) - shared_state.screening_data_factory = screening_unit_factory() + shared_state.screening_data_factory = _screening_unit_factory() shared_state.qualifications += ScreenTaskRequired.get_mixin_qualifications( cfg.mephisto, shared_state, diff --git a/examples/form_composer_demo/run_task_with_worker_opinion.py b/examples/form_composer_demo/run_task_with_worker_opinion.py index 0ecf806f9..e0e33344c 100644 --- a/examples/form_composer_demo/run_task_with_worker_opinion.py +++ b/examples/form_composer_demo/run_task_with_worker_opinion.py @@ -9,68 +9,19 @@ from omegaconf import DictConfig from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script @task_script(default_config_file="example_local_mock") def main(operator: Operator, cfg: DictConfig) -> None: os.environ["REACT_APP__WITH_WORKER_OPINION"] = "true" - - # Build packages - _build_custom_bundles(cfg) - - operator.launch_task_run(cfg.mephisto) - operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) - - -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `mephisto-task-addons` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-addons", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:simple:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, + examples.build_form_composer_simple_with_worker_opinion( force_rebuild=cfg.mephisto.task.force_rebuild, post_install_script=cfg.mephisto.task.post_install_script, - build_command="dev:simple", ) + operator.launch_task_run(cfg.mephisto) + operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) if __name__ == "__main__": diff --git a/examples/parlai_chat_task_demo/run_task.py b/examples/parlai_chat_task_demo/run_task.py index fce6a02de..5013b13da 100644 --- a/examples/parlai_chat_task_demo/run_task.py +++ b/examples/parlai_chat_task_demo/run_task.py @@ -4,17 +4,19 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. - import os -from mephisto.operations.operator import Operator -from mephisto.tools.scripts import task_script -from mephisto.operations.hydra_config import build_default_task_config +from dataclasses import dataclass +from dataclasses import field + +from omegaconf import DictConfig + from mephisto.abstractions.blueprints.parlai_chat.parlai_chat_blueprint import ( SharedParlAITaskState, ) - -from omegaconf import DictConfig -from dataclasses import dataclass, field +from mephisto.operations.hydra_config import build_default_task_config +from mephisto.operations.operator import Operator +from mephisto.tools.building_react_apps import examples +from mephisto.tools.scripts import task_script @dataclass @@ -33,6 +35,10 @@ class ParlAITaskConfig(build_default_task_config("example")): # type: ignore @task_script(config=ParlAITaskConfig) def main(operator: "Operator", cfg: DictConfig) -> None: + examples.build_parlai_chat_task_demo( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) world_opt = {"num_turns": cfg.num_turns, "turn_timeout": cfg.turn_timeout} diff --git a/examples/remote_procedure/mnist/model.py b/examples/remote_procedure/mnist/model.py index 7e2299cb2..57fc46699 100644 --- a/examples/remote_procedure/mnist/model.py +++ b/examples/remote_procedure/mnist/model.py @@ -7,17 +7,22 @@ # Using the implementation provided at # https://github.com/aaron-xichen/pytorch-playground/blob/master/mnist/model.py -import torch.nn as nn from collections import OrderedDict + +import torch.nn as nn import torch.utils.model_zoo as model_zoo -model_urls = {"mnist": "http://ml.cs.tsinghua.edu.cn/~chenxi/pytorch-models/mnist-b07bb66b.pth"} +model_urls = { + "mnist": "http://ml.cs.tsinghua.edu.cn/~chenxi/pytorch-models/mnist-b07bb66b.pth", +} class MLP(nn.Module): def __init__(self, input_dims, n_hiddens, n_class): super(MLP, self).__init__() + assert isinstance(input_dims, int), "Please provide int for input_dims" + self.input_dims = input_dims current_dims = input_dims layers = OrderedDict() @@ -26,31 +31,38 @@ def __init__(self, input_dims, n_hiddens, n_class): n_hiddens = [n_hiddens] else: n_hiddens = list(n_hiddens) + for i, n_hidden in enumerate(n_hiddens): layers["fc{}".format(i + 1)] = nn.Linear(current_dims, n_hidden) layers["relu{}".format(i + 1)] = nn.ReLU() layers["drop{}".format(i + 1)] = nn.Dropout(0.2) current_dims = n_hidden + layers["out"] = nn.Linear(current_dims, n_class) self.model = nn.Sequential(layers) print(self.model) - def forward(self, input): + def forward(self, _input): # input = input.view(input.size(0), -1) - assert input.size(1) == self.input_dims - return self.model.forward(input) + assert _input.size(1) == self.input_dims + + return self.model.forward(_input) -def mnist(input_dims=784, n_hiddens=[256, 256], n_class=10, pretrained=None): +def mnist(input_dims=784, n_hiddens=[256, 256], n_class=10, pretrained=None) -> MLP: model = MLP(input_dims, n_hiddens, n_class) + if pretrained is not None: try: m = model_zoo.load_url(model_urls["mnist"]) except RuntimeError: # No GPU, CPU only m = model_zoo.load_url(model_urls["mnist"], map_location="cpu") + state_dict = m.state_dict() if isinstance(m, nn.Module) else m assert isinstance(state_dict, (dict, OrderedDict)), type(state_dict) + model.load_state_dict(state_dict) + return model diff --git a/examples/remote_procedure/mnist/run_task.py b/examples/remote_procedure/mnist/run_task.py index 8446aee38..ac72b9cd4 100644 --- a/examples/remote_procedure/mnist/run_task.py +++ b/examples/remote_procedure/mnist/run_task.py @@ -9,34 +9,39 @@ from PIL import Image except ImportError: print( - "Need to have torch, PIL, numpy installed to use this demo. For example: pip install torch pillow numpy" + "Need to have torch, PIL, numpy installed to use this demo. " + "For example: pip install torch pillow numpy" ) exit(1) import base64 from io import BytesIO +from typing import Any +from typing import Dict +from typing import List + +from omegaconf import DictConfig + +from examples.remote_procedure.mnist.model import mnist from mephisto.abstractions.blueprints.mixins.screen_task_required import ( ScreenTaskRequired, ) -from mephisto.data_model.unit import Unit -from model import mnist - -from mephisto.operations.operator import Operator -from mephisto.tools.scripts import ( - build_custom_bundle, - task_script, +from mephisto.abstractions.blueprints.remote_procedure.remote_procedure_blueprint import ( + RemoteProcedureAgentState, ) from mephisto.abstractions.blueprints.remote_procedure.remote_procedure_blueprint import ( SharedRemoteProcedureTaskState, - RemoteProcedureAgentState, ) +from mephisto.data_model.unit import Unit +from mephisto.operations.operator import Operator +from mephisto.tools.building_react_apps import examples +from mephisto.tools.scripts import task_script +from mephisto.utils.console_writer import ConsoleWriter -from omegaconf import DictConfig -from typing import List, Any, Dict -from rich import print +logger = ConsoleWriter() -def my_screening_unit_generator(): +def _my_screening_unit_generator() -> dict: """ The frontend react webapp reads in isScreeningUnit using the initialTaskData @@ -48,7 +53,7 @@ def my_screening_unit_generator(): } -def validate_screening_unit(unit: Unit): +def _validate_screening_unit(unit: Unit) -> bool: """Checking if the drawn number is 3""" agent = unit.get_assigned_agent() if agent is not None: @@ -60,32 +65,45 @@ def validate_screening_unit(unit: Unit): return False +def _handle_with_model( + _request_id: str, + args: Dict[str, Any], + agent_state: RemoteProcedureAgentState, +) -> Dict[str, Any]: + """Convert the image to be read by MNIST classifier, then classify""" + + img_dat = args["urlData"].split("data:image/png;base64,")[1] + im = Image.open(BytesIO(base64.b64decode(img_dat))) + im_gray = im.convert("L") + im_resized = im_gray.resize((28, 28)) + im_vals = list(im_resized.getdata()) + norm_vals = [(255 - x) * 1.0 / 255.0 for x in im_vals] + in_tensor = torch.tensor([norm_vals]) + + mnist_model = mnist(pretrained=True) + output = mnist_model(in_tensor) + + pred = output.data.max(1)[1] + + logger.debug(f"Predicted digit: {pred.item()}") + + return { + "digit_prediction": pred.item(), + } + + @task_script(default_config_file="example_local_mock") def main(operator: Operator, cfg: DictConfig) -> None: + examples.build_remote_procedure_mnist( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) + tasks: List[Dict[str, Any]] = [{"isScreeningUnit": False}] * cfg.num_tasks - mnist_model = mnist(pretrained=True) is_using_screening_units = cfg.mephisto.blueprint["use_screening_task"] - def handle_with_model( - _request_id: str, args: Dict[str, Any], agent_state: RemoteProcedureAgentState - ) -> Dict[str, Any]: - """Convert the image to be read by MNIST classifier, then classify""" - img_dat = args["urlData"].split("data:image/png;base64,")[1] - im = Image.open(BytesIO(base64.b64decode(img_dat))) - im_gray = im.convert("L") - im_resized = im_gray.resize((28, 28)) - im_vals = list(im_resized.getdata()) - norm_vals = [(255 - x) * 1.0 / 255.0 for x in im_vals] - in_tensor = torch.tensor([norm_vals]) - output = mnist_model(in_tensor) - pred = output.data.max(1)[1] - print("Predicted digit:", pred.item()) - return { - "digit_prediction": pred.item(), - } - function_registry = { - "classify_digit": handle_with_model, + "classify_digit": _handle_with_model, } shared_state = SharedRemoteProcedureTaskState( static_task_data=tasks, @@ -96,21 +114,13 @@ def handle_with_model( # You have to define a few more properties to enable screening units shared_state.on_unit_submitted = ScreenTaskRequired.create_validation_function( cfg.mephisto, - validate_screening_unit, + _validate_screening_unit, ) - shared_state.screening_data_factory = my_screening_unit_generator() + shared_state.screening_data_factory = _my_screening_unit_generator() shared_state.qualifications += ScreenTaskRequired.get_mixin_qualifications( cfg.mephisto, shared_state ) - task_dir = cfg.task_dir - - build_custom_bundle( - task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - ) - operator.launch_task_run(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) diff --git a/examples/remote_procedure/mnist/webapp/package.json b/examples/remote_procedure/mnist/webapp/package.json index 05c0debca..ba91f7c06 100644 --- a/examples/remote_procedure/mnist/webapp/package.json +++ b/examples/remote_procedure/mnist/webapp/package.json @@ -1,5 +1,5 @@ { - "name": "parlai-mturk-task-compiler", + "name": "remote-procedure-mnist-example", "version": "1.0.1", "description": "", "main": "webpack.config.js", diff --git a/examples/remote_procedure/mnist/webapp/src/app.jsx b/examples/remote_procedure/mnist/webapp/src/app.jsx index 4ec316575..729608c76 100644 --- a/examples/remote_procedure/mnist/webapp/src/app.jsx +++ b/examples/remote_procedure/mnist/webapp/src/app.jsx @@ -16,7 +16,7 @@ import { MephistoContext, useMephistoRemoteProcedureTask, ErrorBoundary, -} from "mephisto-task"; +} from "mephisto-task-multipart"; /* ================= Application Components ================= */ diff --git a/examples/remote_procedure/mnist/webapp/webpack.config.js b/examples/remote_procedure/mnist/webapp/webpack.config.js index 7c92f67df..5438d5a60 100644 --- a/examples/remote_procedure/mnist/webapp/webpack.config.js +++ b/examples/remote_procedure/mnist/webapp/webpack.config.js @@ -16,6 +16,11 @@ module.exports = { resolve: { alias: { react: path.resolve("./node_modules/react"), + // Use local library with code that can submit FormData + "mephisto-task-multipart": path.resolve( + __dirname, + "../../../../packages/mephisto-task-multipart" + ), }, fallback: { net: false, diff --git a/examples/remote_procedure/mnist/webapp/webpack.config.review.js b/examples/remote_procedure/mnist/webapp/webpack.config.review.js index 5e226ee6c..68bd1bc4f 100644 --- a/examples/remote_procedure/mnist/webapp/webpack.config.review.js +++ b/examples/remote_procedure/mnist/webapp/webpack.config.review.js @@ -16,6 +16,11 @@ module.exports = { resolve: { alias: { react: path.resolve("./node_modules/react"), + // Use local library with code that can submit FormData + "mephisto-task-multipart": path.resolve( + __dirname, + "../../../../packages/mephisto-task-multipart" + ), }, fallback: { net: false, diff --git a/examples/remote_procedure/template/run_task.py b/examples/remote_procedure/template/run_task.py index ef4642f42..e1b54f4c2 100644 --- a/examples/remote_procedure/template/run_task.py +++ b/examples/remote_procedure/template/run_task.py @@ -4,21 +4,27 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -from mephisto.operations.operator import Operator -from mephisto.tools.scripts import ( - task_script, - build_custom_bundle, +from typing import Any +from typing import Dict +from typing import List + +from omegaconf import DictConfig + +from mephisto.abstractions.blueprints.remote_procedure.remote_procedure_blueprint import ( + RemoteProcedureAgentState, ) from mephisto.abstractions.blueprints.remote_procedure.remote_procedure_blueprint import ( SharedRemoteProcedureTaskState, - RemoteProcedureAgentState, ) +from mephisto.operations.operator import Operator +from mephisto.tools.building_react_apps import examples +from mephisto.tools.scripts import task_script +from mephisto.utils.console_writer import ConsoleWriter -from omegaconf import DictConfig -from typing import Any, Dict +logger = ConsoleWriter() -def build_local_context(num_tasks): +def _build_local_context(num_tasks: int) -> dict: """ Create local context that you don't intend to be shared with the frontend, but which you may want your remote functions to use @@ -28,10 +34,11 @@ def build_local_context(num_tasks): context = {} for x in range(num_tasks): context[x] = f"Hello {x}, this task was for you." + return context -def build_tasks(num_tasks): +def _build_tasks(num_tasks: int) -> List[dict]: """ Create a set of tasks you want annotated """ @@ -44,52 +51,59 @@ def build_tasks(num_tasks): "local_value_key": x, } ) + return tasks +def _onboarding_always_valid(onboarding_data: dict) -> bool: + # NOTE you can make an onboarding task and validate it here + logger.debug(f"Onboarding data: {onboarding_data}") + return True + + @task_script(default_config_file="example_local_mock") def main(operator: Operator, cfg: DictConfig) -> None: - def onboarding_always_valid(onboarding_data): - # NOTE you can make an onboarding task and validate it here - print(onboarding_data) - return True + examples.build_remote_procedure_template( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) # Right now we're building locally, but should eventually # use non-local for the real thing - tasks = build_tasks(cfg.num_tasks) - context = build_local_context(cfg.num_tasks) + tasks = _build_tasks(cfg.num_tasks) + context = _build_local_context(cfg.num_tasks) - def handle_with_model( - _request_id: str, args: Dict[str, Any], agent_state: RemoteProcedureAgentState + def _handle_with_model( + _request_id: str, + args: Dict[str, Any], + agent_state: RemoteProcedureAgentState, ) -> Dict[str, Any]: """Remote call to process external content using a 'model'""" # NOTE this body can be whatever you want - print(f"The parsed args are {args}, you can do what you want with that") - print(f"You can also use {agent_state.init_data}, to get task keys") + logger.debug(f"The parsed args are {args}, you can do what you want with that") + logger.debug(f"You can also use {agent_state.init_data}, to get task keys") + assert agent_state.init_data is not None + idx = agent_state.init_data["local_value_key"] - print(f"And that may let you get local context, like {context[idx]}") + + logger.debug(f"And that may let you get local context, like {context[idx]}") + return { "secret_local_value": context[idx], "update": f"this was request {args['arg3'] + 1}", } function_registry = { - "handle_with_model": handle_with_model, + "handle_with_model": _handle_with_model, } shared_state = SharedRemoteProcedureTaskState( static_task_data=tasks, - validate_onboarding=onboarding_always_valid, + validate_onboarding=_onboarding_always_valid, function_registry=function_registry, ) - task_dir = cfg.task_dir - build_custom_bundle( - task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - ) operator.launch_task_run(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) diff --git a/examples/remote_procedure/template/webapp/package.json b/examples/remote_procedure/template/webapp/package.json index 9e1055d88..df44ad125 100644 --- a/examples/remote_procedure/template/webapp/package.json +++ b/examples/remote_procedure/template/webapp/package.json @@ -1,5 +1,5 @@ { - "name": "parlai-mturk-task-compiler", + "name": "remote-procedure-template-example", "version": "1.0.1", "description": "", "main": "webpack.config.js", diff --git a/examples/remote_procedure/template/webapp/src/app.jsx b/examples/remote_procedure/template/webapp/src/app.jsx index d05ed8427..edc28c90c 100644 --- a/examples/remote_procedure/template/webapp/src/app.jsx +++ b/examples/remote_procedure/template/webapp/src/app.jsx @@ -12,7 +12,7 @@ import { MephistoContext, useMephistoRemoteProcedureTask, ErrorBoundary, -} from "mephisto-task"; +} from "mephisto-task-multipart"; /* ================= Application Components ================= */ diff --git a/examples/remote_procedure/template/webapp/webpack.config.js b/examples/remote_procedure/template/webapp/webpack.config.js index 7c92f67df..5438d5a60 100644 --- a/examples/remote_procedure/template/webapp/webpack.config.js +++ b/examples/remote_procedure/template/webapp/webpack.config.js @@ -16,6 +16,11 @@ module.exports = { resolve: { alias: { react: path.resolve("./node_modules/react"), + // Use local library with code that can submit FormData + "mephisto-task-multipart": path.resolve( + __dirname, + "../../../../packages/mephisto-task-multipart" + ), }, fallback: { net: false, diff --git a/examples/remote_procedure/toxicity_detection/run_task.py b/examples/remote_procedure/toxicity_detection/run_task.py index 681e10576..6deb6586a 100644 --- a/examples/remote_procedure/toxicity_detection/run_task.py +++ b/examples/remote_procedure/toxicity_detection/run_task.py @@ -10,24 +10,25 @@ print("Need to have detoxify to use this demo. For example: pip install detoxify") exit(1) -from mephisto.operations.operator import Operator -from mephisto.tools.scripts import ( - build_custom_bundle, - task_script, -) +from typing import Any +from typing import Dict +from typing import List + +from omegaconf import DictConfig from mephisto.abstractions.blueprints.remote_procedure.remote_procedure_blueprint import ( - SharedRemoteProcedureTaskState, RemoteProcedureAgentState, ) -from omegaconf import DictConfig -from typing import Any, Dict +from mephisto.abstractions.blueprints.remote_procedure.remote_procedure_blueprint import ( + SharedRemoteProcedureTaskState, +) +from mephisto.operations.operator import Operator +from mephisto.tools.building_react_apps import examples +from mephisto.tools.scripts import task_script -def build_tasks(num_tasks): - """ - Create a set of tasks you want annotated - """ +def _build_tasks(num_tasks: int) -> List[dict]: + """Create a set of tasks you want annotated""" # NOTE These form the init_data for a task tasks = [] for x in range(num_tasks): @@ -37,26 +38,35 @@ def build_tasks(num_tasks): "local_value_key": x, } ) + return tasks -def determine_toxicity(text: str): +def _determine_toxicity(text: str) -> str: return Detoxify("original").predict(text)["toxicity"] +def _calculate_toxicity( + _request_id: str, + args: Dict[str, Any], + agent_state: RemoteProcedureAgentState, +) -> Dict[str, Any]: + return { + "toxicity": str(_determine_toxicity(args["text"])), + } + + @task_script(default_config_file="example_local_mock") def main(operator: Operator, cfg: DictConfig) -> None: - tasks = build_tasks(cfg.num_tasks) + examples.build_remote_procedure_toxicity_detection( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) - def calculate_toxicity( - _request_id: str, args: Dict[str, Any], agent_state: RemoteProcedureAgentState - ) -> Dict[str, Any]: - return { - "toxicity": str(determine_toxicity(args["text"])), - } + tasks = _build_tasks(cfg.num_tasks) function_registry = { - "determine_toxicity": calculate_toxicity, + "determine_toxicity": _calculate_toxicity, } shared_state = SharedRemoteProcedureTaskState( @@ -64,13 +74,6 @@ def calculate_toxicity( function_registry=function_registry, ) - task_dir = cfg.task_dir - build_custom_bundle( - task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - ) - operator.launch_task_run(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) diff --git a/examples/remote_procedure/toxicity_detection/webapp/package.json b/examples/remote_procedure/toxicity_detection/webapp/package.json index 90dbf718a..24259b2e8 100644 --- a/examples/remote_procedure/toxicity_detection/webapp/package.json +++ b/examples/remote_procedure/toxicity_detection/webapp/package.json @@ -1,5 +1,5 @@ { - "name": "parlai-mturk-task-compiler", + "name": "remote-procedure-toxicity-detection-example", "version": "1.0.1", "description": "", "main": "webpack.config.js", diff --git a/examples/remote_procedure/toxicity_detection/webapp/src/app.jsx b/examples/remote_procedure/toxicity_detection/webapp/src/app.jsx index 8fde25f54..50111c8ac 100644 --- a/examples/remote_procedure/toxicity_detection/webapp/src/app.jsx +++ b/examples/remote_procedure/toxicity_detection/webapp/src/app.jsx @@ -16,7 +16,7 @@ import { MephistoContext, useMephistoRemoteProcedureTask, ErrorBoundary, -} from "mephisto-task"; +} from "mephisto-task-multipart"; function RemoteProcedureApp() { let mephistoProps = useMephistoRemoteProcedureTask({}); diff --git a/examples/remote_procedure/toxicity_detection/webapp/webpack.config.js b/examples/remote_procedure/toxicity_detection/webapp/webpack.config.js index 7c92f67df..5438d5a60 100644 --- a/examples/remote_procedure/toxicity_detection/webapp/webpack.config.js +++ b/examples/remote_procedure/toxicity_detection/webapp/webpack.config.js @@ -16,6 +16,11 @@ module.exports = { resolve: { alias: { react: path.resolve("./node_modules/react"), + // Use local library with code that can submit FormData + "mephisto-task-multipart": path.resolve( + __dirname, + "../../../../packages/mephisto-task-multipart" + ), }, fallback: { net: false, diff --git a/examples/static_react_task/run_task.py b/examples/static_react_task/run_task.py index 643efed65..5f438a699 100644 --- a/examples/static_react_task/run_task.py +++ b/examples/static_react_task/run_task.py @@ -15,11 +15,11 @@ ) from mephisto.data_model.unit import Unit from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script -def my_screening_unit_generator(): +def _my_screening_unit_generator() -> dict: while True: yield { "text": "SCREENING UNIT: Press the red button", @@ -27,11 +27,12 @@ def my_screening_unit_generator(): } -def validate_screening_unit(unit: Unit): +def _validate_screening_unit(unit: Unit) -> bool: agent = unit.get_assigned_agent() if agent is not None: data = agent.state.get_data() print(data) + if ( data["outputs"] is not None and "rating" in data["outputs"] @@ -39,36 +40,31 @@ def validate_screening_unit(unit: Unit): ): # User pressed the red button return True + return False -def handle_onboarding(onboarding_data): - if onboarding_data["outputs"]["success"] == True: +def _handle_onboarding(onboarding_data: dict) -> bool: + if onboarding_data["outputs"]["success"] is True: return True + return False -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - build_custom_bundle( - cfg.task_dir, +@task_script(default_config_file="example_local_mock.yaml") +def main(operator: Operator, cfg: DictConfig) -> None: + examples.build_static_react_task( force_rebuild=cfg.mephisto.task.force_rebuild, post_install_script=cfg.mephisto.task.post_install_script, ) - -@task_script(default_config_file="example_local_mock.yaml") -def main(operator: Operator, cfg: DictConfig) -> None: - # Build packages - _build_custom_bundles(cfg) - is_using_screening_units = cfg.mephisto.blueprint["use_screening_task"] shared_state = SharedStaticTaskState( static_task_data=[ {"text": "This text is good text!"}, {"text": "This text is bad text!"}, ], - validate_onboarding=handle_onboarding, + validate_onboarding=_handle_onboarding, ) if is_using_screening_units: @@ -76,9 +72,9 @@ def main(operator: Operator, cfg: DictConfig) -> None: # few more properties set on shared_state shared_state.on_unit_submitted = ScreenTaskRequired.create_validation_function( cfg.mephisto, - validate_screening_unit, + _validate_screening_unit, ) - shared_state.screening_data_factory = my_screening_unit_generator() + shared_state.screening_data_factory = _my_screening_unit_generator() shared_state.qualifications += ScreenTaskRequired.get_mixin_qualifications( cfg.mephisto, shared_state ) diff --git a/examples/static_react_task_with_worker_opinion/run_task.py b/examples/static_react_task_with_worker_opinion/run_task.py index d92eb5765..9de968c73 100644 --- a/examples/static_react_task_with_worker_opinion/run_task.py +++ b/examples/static_react_task_with_worker_opinion/run_task.py @@ -4,65 +4,33 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -import os - from omegaconf import DictConfig from mephisto.abstractions.blueprints.abstract.static_task.static_blueprint import ( SharedStaticTaskState, ) from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import examples from mephisto.tools.scripts import task_script -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "packages", - ) +def _onboarding_always_valid(onboarding_data: dict) -> bool: + return True - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `mephisto-task-addons` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-addons", - build_command="build", - ) - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, +@task_script(default_config_file="example_local_mock") +def main(operator: Operator, cfg: DictConfig) -> None: + examples.build_static_react_task_with_worker_opinion( force_rebuild=cfg.mephisto.task.force_rebuild, post_install_script=cfg.mephisto.task.post_install_script, - build_command="dev", ) - -@task_script(default_config_file="example_local_mock") -def main(operator: Operator, cfg: DictConfig) -> None: - # Build packages - _build_custom_bundles(cfg) - - def onboarding_always_valid(onboarding_data): - return True - shared_state = SharedStaticTaskState( static_task_data=[ {"text": "This text is good text!"}, {"text": "This text is bad text!"}, ], - validate_onboarding=onboarding_always_valid, + validate_onboarding=_onboarding_always_valid, ) operator.launch_task_run(cfg.mephisto, shared_state) diff --git a/mephisto/client/cli_review_app_commands.py b/mephisto/client/cli_review_app_commands.py index 44d2c678e..943646239 100644 --- a/mephisto/client/cli_review_app_commands.py +++ b/mephisto/client/cli_review_app_commands.py @@ -12,8 +12,8 @@ from flask.cli import pass_script_info from flask.cli import ScriptInfo -from mephisto.tools.scripts import build_custom_bundle from mephisto.utils.console_writer import ConsoleWriter +from mephisto.tools.building_react_apps import review_app as _review_app logger = ConsoleWriter() @@ -73,14 +73,7 @@ def review_app( if os.path.exists(os.path.join(client_path, "build", "index.html")) and not force_rebuild: logger.info(f"[blue]React bundle is already built.[/blue]") else: - logger.info(f"[blue]Building React bundle started.[/blue]") - build_custom_bundle( - review_app_path, - force_rebuild=force_rebuild, - webapp_name=client_dir, - build_command="build", - ) - logger.info(f"[blue]Building React bundle finished.[/blue]") + _review_app.build_review_app_ui(force_rebuild=force_rebuild, verbose=True) # Set debug debug = debug if debug is not None else get_debug_flag() diff --git a/mephisto/client/cli_scripts_commands.py b/mephisto/client/cli_scripts_commands.py index 5629d352e..d22626ee8 100644 --- a/mephisto/client/cli_scripts_commands.py +++ b/mephisto/client/cli_scripts_commands.py @@ -21,6 +21,7 @@ import mephisto.scripts.mturk.identify_broken_units as identify_broken_units_mturk import mephisto.scripts.mturk.launch_makeup_hits as launch_makeup_hits_mturk import mephisto.scripts.mturk.print_outstanding_hit_status as soft_block_workers_by_mturk_id_mturk +from mephisto.scripts import tests from mephisto.scripts.local_db import auto_generate_all_docs_reference_md from mephisto.utils.console_writer import ConsoleWriter @@ -49,12 +50,18 @@ "print_outstanding_hit_status", "soft_block_workers_by_mturk_id", ] +TESTS_VALID_SCRIPTS_NAMES = [ + "clean_all_mephisto_react_apps", + "build_all_mephisto_react_apps", + "rebuild_all_mephisto_react_apps", +] VALID_SCRIPT_TYPES = [ - "local_db", + "form_composer", "heroku", + "local_db", "metrics", "mturk", - "form_composer", + "tests", ] logger = ConsoleWriter() @@ -121,6 +128,14 @@ def print_non_markdown_list(items: List[str]): FORM_COMPOSER_VALID_SCRIPTS_NAMES[0]: rebuild_all_apps_form_composer.main, }, }, + "tests": { + "valid_script_names": TESTS_VALID_SCRIPTS_NAMES, + "scripts": { + TESTS_VALID_SCRIPTS_NAMES[0]: tests.react_apps.clean, + TESTS_VALID_SCRIPTS_NAMES[1]: tests.react_apps.build, + TESTS_VALID_SCRIPTS_NAMES[2]: tests.react_apps.rebuild, + }, + }, } if script_name is None or ( diff --git a/mephisto/generators/form_composer/run.py b/mephisto/generators/form_composer/run.py index ba2bbdcc7..ed06d6919 100644 --- a/mephisto/generators/form_composer/run.py +++ b/mephisto/generators/form_composer/run.py @@ -14,14 +14,16 @@ from mephisto.generators.form_composer.config_validation.utils import read_config_file from mephisto.generators.form_composer.remote_procedures import JS_NAME_FUNCTION_MAPPING from mephisto.operations.operator import Operator -from mephisto.tools.scripts import build_custom_bundle +from mephisto.tools.building_react_apps import generators from mephisto.tools.scripts import task_script @task_script(default_config_file="default") def main(operator: Operator, cfg: DictConfig) -> None: - # Build packages - _build_custom_bundles(cfg) + generators.build_form_composer_generator_with_packages( + force_rebuild=cfg.mephisto.task.force_rebuild, + post_install_script=cfg.mephisto.task.post_install_script, + ) # Configure shared state task_data_config_path = os.path.join( @@ -40,49 +42,5 @@ def main(operator: Operator, cfg: DictConfig) -> None: operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) -def _build_custom_bundles(cfg: DictConfig) -> None: - """Locally build bundles that are not available on npm repository""" - mephisto_packages_dir = os.path.join( - # Root project directory - os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - ), - "packages", - ) - - # Build `mephisto-task-multipart` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - # Build `react-form-composer` React package - build_custom_bundle( - mephisto_packages_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="react-form-composer", - build_command="build", - ) - - # Build Review UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - webapp_name="webapp", - build_command="build:review", - ) - - # Build Task UI for the application - build_custom_bundle( - cfg.task_dir, - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - webapp_name="webapp", - build_command="build", - ) - - if __name__ == "__main__": main() diff --git a/mephisto/scripts/form_composer/rebuild_all_apps.py b/mephisto/scripts/form_composer/rebuild_all_apps.py index b5d32fd8a..1ea205fb1 100644 --- a/mephisto/scripts/form_composer/rebuild_all_apps.py +++ b/mephisto/scripts/form_composer/rebuild_all_apps.py @@ -18,227 +18,59 @@ """ import argparse -import os -import shutil -from pathlib import Path -from rich import print +from mephisto.tools.building_react_apps import examples +from mephisto.tools.building_react_apps import generators +from mephisto.tools.building_react_apps import packages +from mephisto.tools.building_react_apps import review_app +from mephisto.utils.console_writer import ConsoleWriter -from mephisto.generators.form_composer.config_validation.utils import set_custom_triggers_js_env_var -from mephisto.generators.form_composer.config_validation.utils import ( - set_custom_validators_js_env_var, -) -from mephisto.tools.scripts import build_custom_bundle +logger = ConsoleWriter() -def _clean_examples_form_composer_demo(repo_path: str, remove_package_locks: bool): - webapp_path = os.path.join( - repo_path, - "examples", - "form_composer_demo", - "webapp", - ) - print(f"[blue]Cleaning '{webapp_path}'[/blue]") - build_path = os.path.join(webapp_path, "build") - node_modules_path = os.path.join(webapp_path, "node_modules") - package_locks_path = os.path.join(webapp_path, "package-lock.json") - shutil.rmtree(build_path, ignore_errors=True) - shutil.rmtree(node_modules_path, ignore_errors=True) - if remove_package_locks: - Path(package_locks_path).unlink(missing_ok=True) +def _clean(remove_package_locks: bool): + verbose = True + if verbose: + logger.info("[blue]Started cleaning up all `build` and `node_modules` directories[/blue]") -def _clean_generators_form_composer(repo_path: str, remove_package_locks: bool): - webapp_path = os.path.join( - repo_path, - "mephisto", - "generators", - "form_composer", - "webapp", - ) - print(f"[blue]Cleaning '{webapp_path}'[/blue]") - build_path = os.path.join(webapp_path, "build") - node_modules_path = os.path.join(webapp_path, "node_modules") - package_locks_path = os.path.join(webapp_path, "package-lock.json") - shutil.rmtree(build_path, ignore_errors=True) - shutil.rmtree(node_modules_path, ignore_errors=True) - if remove_package_locks: - Path(package_locks_path).unlink(missing_ok=True) + packages.clean_mephisto_task_multipart_package(remove_package_locks, verbose=verbose) + packages.clean_mephisto_task_addons_package(remove_package_locks, verbose=verbose) + packages.clean_react_form_composer_package(remove_package_locks, verbose=verbose) + generators.clean_form_composer_generator(remove_package_locks, verbose=verbose) -def _clean_review_app(repo_path: str, remove_package_locks: bool): - webapp_path = os.path.join( - repo_path, - "mephisto", - "review_app", - "client", - ) - print(f"[blue]Cleaning '{webapp_path}'[/blue]") - build_path = os.path.join(webapp_path, "build") - node_modules_path = os.path.join(webapp_path, "node_modules") - package_locks_path = os.path.join(webapp_path, "package-lock.json") - shutil.rmtree(build_path, ignore_errors=True) - shutil.rmtree(node_modules_path, ignore_errors=True) - if remove_package_locks: - Path(package_locks_path).unlink(missing_ok=True) + review_app.clean_review_app(remove_package_locks, verbose=verbose) + examples.clean_form_composer_demo(remove_package_locks, verbose=verbose) -def _clean_packages_mephisto_task_multipart(repo_path: str, remove_package_locks: bool): - webapp_path = os.path.join( - repo_path, - "packages", - "mephisto-task-multipart", - ) - print(f"[blue]Cleaning '{webapp_path}'[/blue]") - build_path = os.path.join(webapp_path, "build") - node_modules_path = os.path.join(webapp_path, "node_modules") - package_locks_path = os.path.join(webapp_path, "package-lock.json") - shutil.rmtree(build_path, ignore_errors=True) - shutil.rmtree(node_modules_path, ignore_errors=True) - if remove_package_locks: - Path(package_locks_path).unlink(missing_ok=True) + if verbose: + logger.info( + "[green]" + "Finished cleaning up all `build` and `node_modules` directories successfully!" + "[/green]" + ) -def _clean_packages_react_form_composer(repo_path: str, remove_package_locks: bool): - webapp_path = os.path.join( - repo_path, - "packages", - "react-form-composer", - ) - print(f"[blue]Cleaning '{webapp_path}'[/blue]") - build_path = os.path.join(webapp_path, "build") - node_modules_path = os.path.join(webapp_path, "node_modules") - package_locks_path = os.path.join(webapp_path, "package-lock.json") - shutil.rmtree(build_path, ignore_errors=True) - shutil.rmtree(node_modules_path, ignore_errors=True) - if remove_package_locks: - Path(package_locks_path).unlink(missing_ok=True) +def _build(): + force_rebuild = True + verbose = True + if verbose: + logger.info("[blue]Started building web apps[/blue]") -def _clean(repo_path: str, remove_package_locks: bool): - print("[blue]Started cleaning up all `build` and `node_modules` directories[/blue]") - _clean_examples_form_composer_demo(repo_path, remove_package_locks) - _clean_generators_form_composer(repo_path, remove_package_locks) - _clean_review_app(repo_path, remove_package_locks) - _clean_packages_mephisto_task_multipart(repo_path, remove_package_locks) - _clean_packages_react_form_composer(repo_path, remove_package_locks) - print( - "[green]" - "Finished cleaning up all `build` and `node_modules` directories successfully!" - "[/green]" - ) + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=verbose) + packages.build_mephisto_task_addons_package(force_rebuild=force_rebuild, verbose=verbose) + packages.build_react_form_composer_package(force_rebuild=force_rebuild, verbose=verbose) + generators.build_form_composer_generator(force_rebuild=force_rebuild, verbose=verbose) -def _build_examples_form_composer_demo(repo_path: str): - app_path = os.path.join( - repo_path, - "examples", - "form_composer_demo", - ) - print(f"[blue]Building '{app_path}'[/blue]") + review_app.build_review_app_ui(force_rebuild=force_rebuild, verbose=verbose) - # Set env var for `custom_validators.js` - from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME + examples.build_form_composer_dynamic(force_rebuild=force_rebuild) - data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME, "dynamic") - set_custom_validators_js_env_var(data_path) - set_custom_triggers_js_env_var(data_path) - - # Build Review UI for the application - build_custom_bundle( - app_path, - force_rebuild=True, - webapp_name="webapp", - build_command="build:review", - ) - - # Build Task UI for the application - build_custom_bundle( - app_path, - force_rebuild=True, - webapp_name="webapp", - build_command="dev", - ) - - -def _build_generators_form_composer(repo_path: str): - app_path = os.path.join( - repo_path, - "mephisto", - "generators", - "form_composer", - ) - print(f"[blue]Building '{app_path}'[/blue]") - - # Set env var for `custom_validators.js` - from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME - - data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME) - set_custom_validators_js_env_var(data_path) - set_custom_triggers_js_env_var(data_path) - - # Build Review UI for the application - build_custom_bundle( - app_path, - force_rebuild=True, - webapp_name="webapp", - build_command="build:review", - ) - - # Build Task UI for the application - build_custom_bundle( - app_path, - force_rebuild=True, - webapp_name="webapp", - build_command="build", - ) - - -def _build_review_app(repo_path: str): - app_path = os.path.join( - repo_path, - "mephisto", - "review_app", - ) - print(f"[blue]Building '{app_path}'[/blue]") - build_custom_bundle( - app_path, - force_rebuild=True, - webapp_name="client", - build_command="build", - ) - - -def _build_packages_mephisto_task_multipart(repo_path: str): - webapp_path = os.path.join(repo_path, "packages") - print(f"[blue]Building '{webapp_path}/mephisto-task-multipart'[/blue]") - build_custom_bundle( - webapp_path, - force_rebuild=True, - webapp_name="mephisto-task-multipart", - build_command="build", - ) - - -def _build_packages_react_form_composer(repo_path: str): - webapp_path = os.path.join(repo_path, "packages") - print(f"[blue]Building '{webapp_path}/react-form-composer'[/blue]") - build_custom_bundle( - webapp_path, - force_rebuild=True, - webapp_name="react-form-composer", - build_command="build", - ) - - -def _build(repo_path: str): - print("[blue]Started building web apps[/blue]") - _build_packages_mephisto_task_multipart(repo_path) - _build_packages_react_form_composer(repo_path) - _build_examples_form_composer_demo(repo_path) - _build_generators_form_composer(repo_path) - _build_review_app(repo_path) - print("[green]Finished building web apps successfully![/green]") + if verbose: + logger.info("[green]Finished building web apps successfully![/green]") def main(): @@ -249,12 +81,8 @@ def main(): parser.add_argument("--rebuild-package-locks", action=argparse.BooleanOptionalAction) args = parser.parse_args() - repo_path = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - ) - - _clean(repo_path, remove_package_locks=args.rebuild_package_locks) - _build(repo_path) + _clean(remove_package_locks=args.rebuild_package_locks) + _build() if __name__ == "__main__": diff --git a/mephisto/scripts/tests/__init__.py b/mephisto/scripts/tests/__init__.py new file mode 100644 index 000000000..d92247a86 --- /dev/null +++ b/mephisto/scripts/tests/__init__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from . import react_apps diff --git a/mephisto/scripts/tests/react_apps.py b/mephisto/scripts/tests/react_apps.py new file mode 100644 index 000000000..3587d3d98 --- /dev/null +++ b/mephisto/scripts/tests/react_apps.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import argparse + +from mephisto.tools.building_react_apps import examples +from mephisto.tools.building_react_apps import generators +from mephisto.tools.building_react_apps import packages +from mephisto.tools.building_react_apps import review_app +from mephisto.utils.console_writer import ConsoleWriter + +logger = ConsoleWriter() + + +def _clean(remove_package_locks: bool): + verbose = True + + if verbose: + logger.info("[blue]Started cleaning up all `build` and `node_modules` directories[/blue]") + + # TODO: --- Cannot be built (maybe outdated configs). Fix and uncomment later --- + # packages.clean_annotated_bbox_package(remove_package_locks, verbose=verbose) + # packages.clean_annotated_keypoint_package(remove_package_locks, verbose=verbose) + # packages.clean_annotated_shell_package(remove_package_locks, verbose=verbose) + # packages.clean_annotated_video_player_package(remove_package_locks, verbose=verbose) + # packages.clean_annotation_toolkit_package(remove_package_locks, verbose=verbose) + # packages.clean_mephisto_review_hook_package(remove_package_locks, verbose=verbose) + # TODO: --- End --- + packages.clean_bootstrap_chat_package(remove_package_locks, verbose=verbose) + packages.clean_global_context_store_package(remove_package_locks, verbose=verbose) + packages.clean_mephisto_task_package(remove_package_locks, verbose=verbose) + packages.clean_mephisto_task_multipart_package(remove_package_locks, verbose=verbose) + packages.clean_mephisto_task_addons_package(remove_package_locks, verbose=verbose) + packages.clean_mephisto_worker_addons_package(remove_package_locks, verbose=verbose) + packages.clean_react_form_composer_package(remove_package_locks, verbose=verbose) + + generators.clean_form_composer_generator(remove_package_locks, verbose=verbose) + + review_app.clean_review_app(remove_package_locks, verbose=verbose) + + examples.clean_form_composer_demo(remove_package_locks, verbose=verbose) + examples.clean_parlai_chat_task_demo(remove_package_locks, verbose=verbose) + examples.clean_remote_procedure_mnist(remove_package_locks, verbose=verbose) + examples.clean_remote_procedure_template(remove_package_locks, verbose=verbose) + examples.clean_remote_procedure_toxicity_detection(remove_package_locks, verbose=verbose) + examples.clean_static_react_task(remove_package_locks, verbose=verbose) + examples.clean_static_react_task_with_worker_opinion(remove_package_locks, verbose=verbose) + + if verbose: + logger.info( + "[green]" + "Finished cleaning up all `build` and `node_modules` directories successfully!" + "[/green]" + ) + + +def _build(): + force_rebuild = True + verbose = True + + if verbose: + logger.info("[blue]Started building web apps[/blue]") + + # TODO: --- Cannot be built (maybe outdated configs). Fix and uncomment later --- + # packages.build_annotated_bbox_package(force_rebuild=force_rebuild, verbose=verbose) + # packages.build_annotated_keypoint_package(force_rebuild=force_rebuild, verbose=verbose) + # packages.build_annotated_shell_package(force_rebuild=force_rebuild, verbose=verbose) + # packages.build_annotated_video_player_package(force_rebuild=force_rebuild, verbose=verbose) + # packages.build_annotation_toolkit_package(force_rebuild=force_rebuild, verbose=verbose) + # packages.build_mephisto_review_hook_package(force_rebuild=force_rebuild, verbose=verbose) + # TODO: --- End --- + packages.build_bootstrap_chat_package(force_rebuild=force_rebuild, verbose=verbose) + packages.build_global_context_store_package(force_rebuild=force_rebuild, verbose=verbose) + packages.build_mephisto_task_package(force_rebuild=force_rebuild, verbose=verbose) + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=verbose) + packages.build_mephisto_task_addons_package(force_rebuild=force_rebuild, verbose=verbose) + packages.build_mephisto_worker_addons_package(force_rebuild=force_rebuild, verbose=verbose) + packages.build_react_form_composer_package(force_rebuild=force_rebuild, verbose=verbose) + + generators.build_form_composer_generator(force_rebuild=force_rebuild, verbose=verbose) + + review_app.build_review_app_ui(force_rebuild=force_rebuild, verbose=verbose) + + examples.build_form_composer_simple(force_rebuild=force_rebuild) + examples.build_form_composer_simple_with_worker_opinion(force_rebuild=force_rebuild) + examples.build_form_composer_dynamic(force_rebuild=force_rebuild) + examples.build_form_composer_dynamic_presigned_urls_ec2_prolific(force_rebuild=force_rebuild) + examples.build_parlai_chat_task_demo(force_rebuild=force_rebuild) + examples.build_remote_procedure_mnist(force_rebuild=force_rebuild) + examples.build_remote_procedure_template(force_rebuild=force_rebuild) + examples.build_remote_procedure_toxicity_detection(force_rebuild=force_rebuild) + examples.build_static_react_task(force_rebuild=force_rebuild) + examples.build_static_react_task_with_worker_opinion(force_rebuild=force_rebuild) + + if verbose: + logger.info("[green]Finished building web apps successfully![/green]") + + +def _parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("scripts") + parser.add_argument("tests") + parser.add_argument("clean") + parser.add_argument("--rebuild-package-locks", action=argparse.BooleanOptionalAction) + args = parser.parse_args() + return args + + +def clean(): + args = _parse_arguments() + _clean(remove_package_locks=args.rebuild_package_locks) + + +def build(): + _build() + + +def rebuild(): + clean() + build() + + +if __name__ == "__main__": + rebuild() diff --git a/mephisto/tools/building_react_apps/__init__.py b/mephisto/tools/building_react_apps/__init__.py new file mode 100644 index 000000000..cfaca7562 --- /dev/null +++ b/mephisto/tools/building_react_apps/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. diff --git a/mephisto/tools/building_react_apps/examples.py b/mephisto/tools/building_react_apps/examples.py new file mode 100644 index 000000000..0ae390b9a --- /dev/null +++ b/mephisto/tools/building_react_apps/examples.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import os +from typing import Optional + +from mephisto.generators.form_composer.config_validation.utils import set_custom_triggers_js_env_var +from mephisto.generators.form_composer.config_validation.utils import ( + set_custom_validators_js_env_var, +) +from mephisto.tools.scripts import build_custom_bundle +from mephisto.utils.console_writer import ConsoleWriter +from . import packages +from .utils import clean_single_react_app + +REPO_PATH = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +EXAMPLES_PATH = os.path.join(REPO_PATH, "examples") +FORM_COMPOSER_EXAMPLE_PATH = os.path.join(EXAMPLES_PATH, "form_composer_demo") +PARLAI_CHAT_TASK_EXAMPLE_PATH = os.path.join(EXAMPLES_PATH, "parlai_chat_task_demo") +REMOTE_PROCEDURE_MNIST_EXAMPLE_PATH = os.path.join(EXAMPLES_PATH, "remote_procedure", "mnist") +REMOTE_PROCEDURE_TEMPLATE_EXAMPLE_PATH = os.path.join(EXAMPLES_PATH, "remote_procedure", "template") +REMOTE_PROCEDURE_TOXICITY_DETECTION_EXAMPLE_PATH = os.path.join( + EXAMPLES_PATH, + "remote_procedure", + "toxicity_detection", +) +STATIC_REACT_TASK_EXAMPLE_PATH = os.path.join(EXAMPLES_PATH, "static_react_task") +STATIC_REACT_TASK_WITH_WORKER_OPINION_EXAMPLE_PATH = os.path.join( + EXAMPLES_PATH, + "static_react_task_with_worker_opinion", +) + +logger = ConsoleWriter() + + +# --- Form Composer --- + + +def clean_form_composer_demo(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(FORM_COMPOSER_EXAMPLE_PATH, "webapp") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def build_form_composer_simple( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=True) + packages.build_react_form_composer_package(force_rebuild=force_rebuild, verbose=True) + + # Build Review UI for the application + build_custom_bundle( + FORM_COMPOSER_EXAMPLE_PATH, + force_rebuild=force_rebuild, + webapp_name="webapp", + build_command="build:simple:review", + ) + + # Build Task UI for the application + build_custom_bundle( + FORM_COMPOSER_EXAMPLE_PATH, + force_rebuild=force_rebuild, + post_install_script=post_install_script, + build_command="dev:simple", + ) + + +def build_form_composer_simple_with_gold_unit( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + build_form_composer_simple( + force_rebuild=force_rebuild, + post_install_script=post_install_script, + ) + + +def build_form_composer_simple_with_onboarding( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + build_form_composer_simple( + force_rebuild=force_rebuild, + post_install_script=post_install_script, + ) + + +def build_form_composer_simple_with_screening( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + build_form_composer_simple( + force_rebuild=force_rebuild, + post_install_script=post_install_script, + ) + + +def build_form_composer_simple_with_worker_opinion( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=True) + packages.build_mephisto_task_addons_package(force_rebuild=force_rebuild, verbose=True) + packages.build_react_form_composer_package(force_rebuild=force_rebuild, verbose=True) + + # Build Review UI for the application + build_custom_bundle( + FORM_COMPOSER_EXAMPLE_PATH, + force_rebuild=force_rebuild, + webapp_name="webapp", + build_command="build:simple:review", + ) + + # Build Task UI for the application + build_custom_bundle( + FORM_COMPOSER_EXAMPLE_PATH, + force_rebuild=force_rebuild, + post_install_script=post_install_script, + build_command="dev:simple", + ) + + +def build_form_composer_dynamic( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=True) + packages.build_react_form_composer_package(force_rebuild=force_rebuild, verbose=True) + + # Set env vars for `custom_validators.js` and `custom_triggers.js` + from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME + + data_path = os.path.join(FORM_COMPOSER_EXAMPLE_PATH, FORM_COMPOSER__DATA_DIR_NAME, "dynamic") + set_custom_validators_js_env_var(data_path) + set_custom_triggers_js_env_var(data_path) + + # Build Review UI for the application + build_custom_bundle( + FORM_COMPOSER_EXAMPLE_PATH, + force_rebuild=force_rebuild, + webapp_name="webapp", + build_command="build:review", + ) + + # Build Task UI for the application + build_custom_bundle( + FORM_COMPOSER_EXAMPLE_PATH, + force_rebuild=force_rebuild, + webapp_name="webapp", + post_install_script=post_install_script, + build_command="dev", + ) + + +def build_form_composer_dynamic_ec2_mturk_sandbox( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + build_form_composer_dynamic( + force_rebuild=force_rebuild, + post_install_script=post_install_script, + ) + + +def build_form_composer_dynamic_ec2_prolific( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + build_form_composer_dynamic( + force_rebuild=force_rebuild, + post_install_script=post_install_script, + ) + + +def build_form_composer_dynamic_presigned_urls_ec2_prolific( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=True) + packages.build_react_form_composer_package(force_rebuild=force_rebuild, verbose=True) + + # Build Review UI for the application + build_custom_bundle( + FORM_COMPOSER_EXAMPLE_PATH, + force_rebuild=force_rebuild, + webapp_name="webapp", + build_command="build:review", + ) + + # Build Task UI for the application + build_custom_bundle( + FORM_COMPOSER_EXAMPLE_PATH, + force_rebuild=force_rebuild, + post_install_script=post_install_script, + build_command="build:presigned_urls", + ) + + +# --- Parlai Chat --- + + +def clean_parlai_chat_task_demo(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PARLAI_CHAT_TASK_EXAMPLE_PATH, "webapp") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def build_parlai_chat_task_demo( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + # Build Task UI for the application + build_custom_bundle( + PARLAI_CHAT_TASK_EXAMPLE_PATH, + force_rebuild=force_rebuild, + webapp_name="webapp", + post_install_script=post_install_script, + build_command="dev", + ) + + +# --- Remote Procedure --- + + +def clean_remote_procedure_mnist(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(REMOTE_PROCEDURE_MNIST_EXAMPLE_PATH, "webapp") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_remote_procedure_template(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(REMOTE_PROCEDURE_TEMPLATE_EXAMPLE_PATH, "webapp") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_remote_procedure_toxicity_detection(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(REMOTE_PROCEDURE_TOXICITY_DETECTION_EXAMPLE_PATH, "webapp") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def build_remote_procedure_mnist( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=True) + + # Build Review UI for the application + build_custom_bundle( + REMOTE_PROCEDURE_MNIST_EXAMPLE_PATH, + force_rebuild=force_rebuild, + webapp_name="webapp", + build_command="build:review", + ) + + # Build Task UI for the application + build_custom_bundle( + REMOTE_PROCEDURE_MNIST_EXAMPLE_PATH, + force_rebuild=force_rebuild, + post_install_script=post_install_script, + build_command="dev", + ) + + +def build_remote_procedure_template( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=True) + + # Build Task UI for the application + build_custom_bundle( + REMOTE_PROCEDURE_TEMPLATE_EXAMPLE_PATH, + force_rebuild=force_rebuild, + post_install_script=post_install_script, + build_command="dev", + ) + + +def build_remote_procedure_toxicity_detection( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=True) + + # Build Task UI for the application + build_custom_bundle( + REMOTE_PROCEDURE_TOXICITY_DETECTION_EXAMPLE_PATH, + force_rebuild=force_rebuild, + post_install_script=post_install_script, + build_command="dev", + ) + + +# --- Static React Task --- + + +def clean_static_react_task(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(STATIC_REACT_TASK_EXAMPLE_PATH, "webapp") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def build_static_react_task( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=True) + + # Build Task UI for the application + build_custom_bundle( + STATIC_REACT_TASK_EXAMPLE_PATH, + force_rebuild=force_rebuild, + post_install_script=post_install_script, + ) + + +# --- Static React Task with Worker Opinion --- + + +def clean_static_react_task_with_worker_opinion(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(STATIC_REACT_TASK_WITH_WORKER_OPINION_EXAMPLE_PATH, "webapp") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def build_static_react_task_with_worker_opinion( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=True) + packages.build_mephisto_task_addons_package(force_rebuild=force_rebuild, verbose=True) + + # Build Task UI for the application + build_custom_bundle( + STATIC_REACT_TASK_WITH_WORKER_OPINION_EXAMPLE_PATH, + force_rebuild=force_rebuild, + post_install_script=post_install_script, + build_command="dev", + ) diff --git a/mephisto/tools/building_react_apps/generators.py b/mephisto/tools/building_react_apps/generators.py new file mode 100644 index 000000000..ff06c6774 --- /dev/null +++ b/mephisto/tools/building_react_apps/generators.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import os +from typing import Optional + +from mephisto.generators.form_composer.config_validation.utils import set_custom_triggers_js_env_var +from mephisto.generators.form_composer.config_validation.utils import ( + set_custom_validators_js_env_var, +) +from mephisto.tools.scripts import build_custom_bundle +from mephisto.utils.console_writer import ConsoleWriter +from . import packages +from .utils import clean_single_react_app + +REPO_PATH = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +FORM_COMPOSER_GENERATOR_PATH = os.path.join(REPO_PATH, "mephisto", "generators", "form_composer") + +logger = ConsoleWriter() + + +# --- CLEAN --- + + +def clean_form_composer_generator(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(FORM_COMPOSER_GENERATOR_PATH, "webapp") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +# --- BUILD --- + + +def build_form_composer_generator( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, + verbose: bool = False, +): + if verbose: + logger.info(f"[blue]Building '{FORM_COMPOSER_GENERATOR_PATH}'[/blue]") + + # Set env var for `custom_validators.js` + from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME + + data_path = os.path.join(FORM_COMPOSER_GENERATOR_PATH, FORM_COMPOSER__DATA_DIR_NAME) + set_custom_validators_js_env_var(data_path) + set_custom_triggers_js_env_var(data_path) + + # Build Review UI for the application + build_custom_bundle( + FORM_COMPOSER_GENERATOR_PATH, + force_rebuild=force_rebuild, + webapp_name="webapp", + build_command="build:review", + ) + + # Build Task UI for the application + build_custom_bundle( + FORM_COMPOSER_GENERATOR_PATH, + force_rebuild=force_rebuild, + post_install_script=post_install_script, + webapp_name="webapp", + build_command="build", + ) + + +def build_form_composer_generator_with_packages( + force_rebuild: bool = False, + post_install_script: Optional[str] = None, + verbose: bool = False, +) -> None: + packages.build_mephisto_task_multipart_package(force_rebuild=force_rebuild, verbose=verbose) + packages.build_react_form_composer_package(force_rebuild=force_rebuild, verbose=verbose) + build_form_composer_generator( + force_rebuild=force_rebuild, + post_install_script=post_install_script, + verbose=verbose, + ) diff --git a/mephisto/tools/building_react_apps/packages.py b/mephisto/tools/building_react_apps/packages.py new file mode 100644 index 000000000..222fe6239 --- /dev/null +++ b/mephisto/tools/building_react_apps/packages.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import os + +from mephisto.tools.building_react_apps.utils import clean_single_react_app +from mephisto.tools.scripts import build_custom_bundle +from mephisto.utils.console_writer import ConsoleWriter + +REPO_PATH = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +PACKAGES_PATH = os.path.join(REPO_PATH, "packages") +ANNOTATED_PACKAGES_PATH = os.path.join(PACKAGES_PATH, "annotated") + +logger = ConsoleWriter() + + +# --- CLEAN --- + + +def clean_annotated_bbox_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(ANNOTATED_PACKAGES_PATH, "bbox") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_annotated_keypoint_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(ANNOTATED_PACKAGES_PATH, "keypoint") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_annotated_shell_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(ANNOTATED_PACKAGES_PATH, "shell") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_annotated_video_player_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(ANNOTATED_PACKAGES_PATH, "video-player") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_annotation_toolkit_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PACKAGES_PATH, "annotation-toolkit") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_bootstrap_chat_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PACKAGES_PATH, "bootstrap-chat") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_global_context_store_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PACKAGES_PATH, "global-context-store") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_mephisto_review_hook_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PACKAGES_PATH, "mephisto-review-hook") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_mephisto_task_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PACKAGES_PATH, "mephisto-task") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_mephisto_task_multipart_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PACKAGES_PATH, "mephisto-task-multipart") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_mephisto_task_addons_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PACKAGES_PATH, "mephisto-task-addons") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_mephisto_worker_addons_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PACKAGES_PATH, "mephisto-worker-addons") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +def clean_react_form_composer_package(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(PACKAGES_PATH, "react-form-composer") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +# --- BUILD --- + + +def build_annotated_bbox_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{ANNOTATED_PACKAGES_PATH}/bbox'[/blue]") + + build_custom_bundle( + ANNOTATED_PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="bbox", + build_command="build", + ) + + +def build_annotated_keypoint_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{ANNOTATED_PACKAGES_PATH}/keypoint'[/blue]") + + build_custom_bundle( + ANNOTATED_PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="keypoint", + build_command="build", + ) + + +def build_annotated_shell_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{ANNOTATED_PACKAGES_PATH}/shell'[/blue]") + + build_custom_bundle( + ANNOTATED_PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="shell", + build_command="build", + ) + + +def build_annotated_video_player_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{ANNOTATED_PACKAGES_PATH}/video-player'[/blue]") + + build_custom_bundle( + ANNOTATED_PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="video-player", + build_command="build", + ) + + +def build_annotation_toolkit_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{PACKAGES_PATH}/annotation-toolkit'[/blue]") + + build_custom_bundle( + PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="annotation-toolkit", + build_command="build", + ) + + +def build_bootstrap_chat_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{PACKAGES_PATH}/bootstrap-chat'[/blue]") + + build_custom_bundle( + PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="bootstrap-chat", + build_command="build", + ) + + +def build_global_context_store_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{PACKAGES_PATH}/global-context-store'[/blue]") + + build_custom_bundle( + PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="global-context-store", + build_command="build", + ) + + +def build_mephisto_review_hook_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{PACKAGES_PATH}/mephisto-review-hook'[/blue]") + + build_custom_bundle( + PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="mephisto-review-hook", + build_command="build", + ) + + +def build_mephisto_task_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{PACKAGES_PATH}/mephisto-task'[/blue]") + + build_custom_bundle( + PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="mephisto-task", + build_command="build", + ) + + +def build_mephisto_task_multipart_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{PACKAGES_PATH}/mephisto-task-multipart'[/blue]") + + build_custom_bundle( + PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="mephisto-task-multipart", + build_command="build", + ) + + +def build_mephisto_task_addons_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{PACKAGES_PATH}/mephisto-task-addons'[/blue]") + + build_custom_bundle( + PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="mephisto-task-addons", + build_command="build", + ) + + +def build_mephisto_worker_addons_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{PACKAGES_PATH}/mephisto-worker-addons'[/blue]") + + build_custom_bundle( + PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="mephisto-worker-addons", + build_command="build", + ) + + +def build_react_form_composer_package(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{PACKAGES_PATH}/react-form-composer'[/blue]") + + build_custom_bundle( + PACKAGES_PATH, + force_rebuild=force_rebuild, + webapp_name="react-form-composer", + build_command="build", + ) diff --git a/mephisto/tools/building_react_apps/review_app.py b/mephisto/tools/building_react_apps/review_app.py new file mode 100644 index 000000000..42c807504 --- /dev/null +++ b/mephisto/tools/building_react_apps/review_app.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import os + +from mephisto.tools.scripts import build_custom_bundle +from mephisto.utils.console_writer import ConsoleWriter +from .utils import clean_single_react_app + +REPO_PATH = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +REVIEW_APP_PATH = os.path.join(REPO_PATH, "mephisto", "review_app") + +logger = ConsoleWriter() + + +# --- CLEAN --- + + +def clean_review_app(remove_package_locks: bool, verbose: bool = False): + webapp_path = os.path.join(REVIEW_APP_PATH, "client") + clean_single_react_app(webapp_path, remove_package_locks=remove_package_locks, verbose=verbose) + + +# --- BUILD --- + + +def build_review_app_ui(force_rebuild: bool = False, verbose: bool = False): + if verbose: + logger.info(f"[blue]Building '{REVIEW_APP_PATH}'[/blue]") + + build_custom_bundle( + REVIEW_APP_PATH, + force_rebuild=force_rebuild, + webapp_name="client", + build_command="build", + ) diff --git a/mephisto/tools/building_react_apps/utils.py b/mephisto/tools/building_react_apps/utils.py new file mode 100644 index 000000000..0022cde3c --- /dev/null +++ b/mephisto/tools/building_react_apps/utils.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import os +import shutil +from pathlib import Path + +from mephisto.utils.console_writer import ConsoleWriter + +logger = ConsoleWriter() + + +def clean_single_react_app( + webapp_path: str, + remove_package_locks: bool = False, + verbose: bool = False, +): + if verbose: + logger.info(f"[blue]Cleaning '{webapp_path}'[/blue]") + + build_path = os.path.join(webapp_path, "build") + node_modules_path = os.path.join(webapp_path, "node_modules") + package_locks_path = os.path.join(webapp_path, "package-lock.json") + shutil.rmtree(build_path, ignore_errors=True) + shutil.rmtree(node_modules_path, ignore_errors=True) + + if remove_package_locks: + Path(package_locks_path).unlink(missing_ok=True)