From bf51ed8994907cdccd19ad90afd9a08dd1edbc4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Fernanda=20Magallanes?= <35668326+MaferMazu@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:33:54 -0500 Subject: [PATCH] feat: add picasso commands in simple way (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add picasso commands in simple way * test: add workflows and fix tests * docs: add information in the readme * test: fix tests * fix: add importlib_resources for palm support * fix: add commands in picasso installation * feat: add backward compatibility * fix: fix tests * fix: add condition before MOUNTED_DIRECTORIES * test: add integration tests * fix: implement feedback improvements Co-authored-by: Maria Grimaldi * fix: correct test * docs: improve usage description * fix: implement feedback * test: add again the test-types * fix: apply feedback about quick checks and separate methods * docs: update the readme format Co-authored-by: Brayan Cerón <86393372+bra-i-am@users.noreply.github.com> * chore: improve integration tests * refactor: use handler to manage requirement tasks depending on tutor version * refactor: improve the naming in enable_private_packages * docs: improve code docs * refactor: return all command error messages instead * fix: check the commands inside the loop and format * docs: add documentation about the type ignore * fix: use typing Pattern instead of regex one --------- Co-authored-by: Maria Grimaldi Co-authored-by: Brayan Cerón <86393372+bra-i-am@users.noreply.github.com> --- .github/workflows/commitlint.yml | 11 ++ .github/workflows/integration_test.yml | 72 ++++++++ .github/workflows/test.yml | 2 +- MANIFEST.in | 1 + README.rst | 163 +++++++++++++++++- setup.py | 4 +- tutorpicasso/__about__.py | 2 +- tutorpicasso/commands/cli.py | 23 +++ .../commands/enable_private_packages.py | 123 +++++++++++++ tutorpicasso/commands/enable_themes.py | 40 +++++ tutorpicasso/commands/run_extra_commands.py | 143 +++++++++++++++ tutorpicasso/plugin.py | 18 ++ 12 files changed, 593 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/commitlint.yml create mode 100644 .github/workflows/integration_test.yml create mode 100644 tutorpicasso/commands/cli.py create mode 100644 tutorpicasso/commands/enable_private_packages.py create mode 100644 tutorpicasso/commands/enable_themes.py create mode 100644 tutorpicasso/commands/run_extra_commands.py diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml new file mode 100644 index 0000000..3d98fd6 --- /dev/null +++ b/.github/workflows/commitlint.yml @@ -0,0 +1,11 @@ +name: Lint Commit Messages +on: [pull_request] + +jobs: + commitlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: wagoid/commitlint-github-action@v6 diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml new file mode 100644 index 0000000..2032919 --- /dev/null +++ b/.github/workflows/integration_test.yml @@ -0,0 +1,72 @@ +name: Integration Tests +on: [pull_request] + +jobs: + tutor-integration-test: + name: Integration with Tutor + strategy: + matrix: + tutor_version: ["<19.0.0", "<18.0.0", "<17.0.0", "<16.0.0"] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Prepare Tutor & launch + run: | + pip install "tutor${{ matrix.tutor_version }}" + pip install -e . + TUTOR_ROOT="$(pwd)" tutor --version + TUTOR_ROOT="$(pwd)" tutor config save + + - name: Set up SSH + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan github.com >> ~/.ssh/known_hosts + + - name: Enable the tutor plugin + run: | + TUTOR_ROOT="$(pwd)" tutor plugins enable picasso + TUTOR_ROOT="$(pwd)" tutor picasso -h + + - name: Adding data to the config.yml + run: | + cat <> config.yml + PICASSO_MANAGE_DPKG: + name: eox-manage + repo: git@github.com:eduNEXT/eox-manage.git + version: v5.2.0 + PICASSO_THEMES: + - name: endx-saas-themes + repo: git@github.com:eduNEXT/ednx-saas-themes.git + version: master + PICASSO_EXTRA_COMMANDS: + - tutor plugins update + - tutor plugins index add https://raw.githubusercontent.com/eduNEXT/tutor-plugin-indexes/picasso_test/ + - tutor plugins install mfe mfe_extensions aspects + - tutor plugins enable mfe mfe_extensions aspects + - tutor config save + EOF + + - name: Check run-extra-commands + run: | + TUTOR_ROOT="$(pwd)" tutor picasso run-extra-commands + + - name: Check enable-themes + run: | + TUTOR_ROOT="$(pwd)" tutor picasso enable-themes + + - name: Check enable-private-packages + run: | + TUTOR_ROOT="$(pwd)" tutor picasso enable-private-packages + + if grep -q 'eox-manage' env/build/openedx/requirements/private.txt; then + echo "'eox-manage' found in env/build/openedx/requirements/private.txt." + elif grep -q 'eox-manage' env/build/openedx/Dockerfile; then + echo "'eox-manage' found in env/build/openedx/Dockerfile." + else + echo "'eox-manage' not found for the building process." + exit 1 + fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d17724b..31a8e0e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: Run tests on: pull_request: - branches: [master] + branches: [main] jobs: tests: diff --git a/MANIFEST.in b/MANIFEST.in index e938daf..b9d8645 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include tutorpicasso/patches * recursive-include tutorpicasso/templates * +recursive-include tutorpicasso/commands * diff --git a/README.rst b/README.rst index db57883..1202930 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,22 @@ -picasso plugin for `Tutor `__ -######################################################### +Picasso `Tutor`_ Plugin +######################### -picasso plugin for Tutor +|Maintainance Badge| |Test Badge| + +.. |Maintainance Badge| image:: https://img.shields.io/badge/Status-Maintained-brightgreen + :alt: Maintainance Status +.. |Test Badge| image:: https://img.shields.io/github/actions/workflow/status/edunext/tutor-contrib-picasso/.github%2Fworkflows%2Ftests.yml?label=Test + :alt: GitHub Actions Workflow Test Status + +Picasso is a `Tutor`_ plugin that streamlines and automates complex pre-build tasks into a cohesive command. + +Current features include: + +- Adding private requirements: install private packages or dependencies in edx-platform. +- Executing a bundle of commands: run multiple commands in a specific order with a single command. +- Adding themes: manage custom themes to personalize Open edX. + +This plugin is based on https://github.com/eduNEXT/tutor-contrib-edunext-distro Installation @@ -11,15 +26,153 @@ Installation pip install git+https://github.com/eduNEXT/tutor-contrib-picasso -Usage -***** +Enable the plugin +****************** .. code-block:: bash + # To enable the plugin tutor plugins enable picasso + # Show the picasso commands + tutor picasso -h + + +.. note:: + + Please remember to run these commands before you build your images. + + +Compatibility notes +******************* + +This plugin was tested from Olive release. + +Usage +******* + +Enable Private Packages +^^^^^^^^^^^^^^^^^^^^^^^^ + +To enable the installation of private Open edX Django apps, follow these steps: + +1. Add configuration to the configuration file + +First, add the necessary configuration in your Tutor environment's ``config.yml`` file: + +.. code-block:: yaml + + PICASSO__DPKG: + name: + repo: + version: + +.. note:: + + It is needed to use the SSH URL to clone private packages. + +2. Save the configuration with ``tutor config save`` + +3. Run the following command to enable private packages: + +.. code-block:: bash + + # Enable private packages + tutor picasso enable-private-packages + + +This command allows the installation of private Open edX Django apps. It clones the private repository and, through the ``tutor mounts`` command, adds it to the Dockerfile for inclusion in the build process. + +.. warning:: + + For the mount to work correctly and include the package in the Dockerfile, it must be added to a tutor filter ``MOUNTED_DIRECTORIES``. By default, Picasso adds ``eox-*`` packages. If you need to add another private package, don't forget to include this configuration in a Tutor plugin. + + .. code-block:: python + + hooks.Filters.MOUNTED_DIRECTORIES.add_items( + [ + ("openedx", ""), + ] + ) + + +.. note:: + + If you want to use public packages, we recommend using the ``OPEN_EDX_EXTRA_PIP_REQUIREMENTS`` variable in the ``config.yml`` of your Tutor environment. + + +Enable Themes +^^^^^^^^^^^^^^ + +To enable themes in your Tutor environment, follow these steps: + +1. Add the necessary configuration in your Tutor environment's ``config.yml`` file: + +.. code-block:: yaml + + PICASSO_THEMES: + - name: + repo: + version: + - name: + repo: + version: + +.. note:: + + If your theme repository is public, you can also use the HTTPS URL in ``repo``. + +2. Save the configuration with ``tutor config save`` + +3. Run the enable command + +.. code-block:: bash + + # Enable themes + tutor picasso enable-themes + +This command clones your theme repository into the folder that Tutor uses for themes. Documentation available at `Installing custom theme`_ tutorial. + +.. note:: + + Don't forget to add extra configurations in a Tutor plugin if your theme requires it. + + +Run Extra Commands +^^^^^^^^^^^^^^^^^^^ + +To execute a list of Tutor commands in your Tutor environment, follow these steps: + +1. Add the necessary configuration in your Tutor environment's ``config.yml`` file: + +.. code-block:: yaml + + PICASSO_EXTRA_COMMANDS: + - + - + - + - + . + . + . + +2. Save the configuration with ``tutor config save`` + +3. Run the following command + +.. code-block:: bash + + # Run Tutor commands + tutor picasso run-extra-commands + +This command allows you to run a list of Tutor commands. These commands are executed in bash and, for security reasons, are restricted to running only Tutor commands. + License ******* This software is licensed under the terms of the AGPLv3. + + +.. _Tutor: https://docs.tutor.edly.io +.. _Installing custom theme: https://docs.tutor.edly.io/tutorials/theming.html#theming \ No newline at end of file diff --git a/setup.py b/setup.py index 7cd3175..677969d 100644 --- a/setup.py +++ b/setup.py @@ -41,10 +41,10 @@ def load_about(): packages=find_packages(exclude=["tests*"]), include_package_data=True, python_requires=">=3.8", - install_requires=["tutor>=18.0.0,<19.0.0"], + install_requires=["tutor", "importlib_resources", "packaging"], extras_require={ "dev": [ - "tutor[dev]>=18.0.0,<19.0.0", + "tutor[dev]", ] }, entry_points={ diff --git a/tutorpicasso/__about__.py b/tutorpicasso/__about__.py index c6a8b8e..3dc1f76 100644 --- a/tutorpicasso/__about__.py +++ b/tutorpicasso/__about__.py @@ -1 +1 @@ -__version__ = "18.0.0" +__version__ = "0.1.0" diff --git a/tutorpicasso/commands/cli.py b/tutorpicasso/commands/cli.py new file mode 100644 index 0000000..439e03e --- /dev/null +++ b/tutorpicasso/commands/cli.py @@ -0,0 +1,23 @@ +""" +Picasso commands group. +""" + +import click + +from tutorpicasso.commands.enable_private_packages import enable_private_packages +from tutorpicasso.commands.enable_themes import enable_themes +from tutorpicasso.commands.run_extra_commands import run_extra_commands + + +@click.group(help="Run picasso commands") +def picasso() -> None: + """ + Main picasso command group. + + This command group provides functionality to run picasso commands. + """ + + +picasso.add_command(enable_private_packages) +picasso.add_command(run_extra_commands) +picasso.add_command(enable_themes) diff --git a/tutorpicasso/commands/enable_private_packages.py b/tutorpicasso/commands/enable_private_packages.py new file mode 100644 index 0000000..1036355 --- /dev/null +++ b/tutorpicasso/commands/enable_private_packages.py @@ -0,0 +1,123 @@ +import os +import subprocess +from typing import Any, Dict + +import click +from packaging.version import Version +from tutor import config as tutor_config +from tutor.__about__ import __version__ as tutor_version + + +@click.command(name="enable-private-packages", help="Enable picasso private packages") +def enable_private_packages() -> None: + """ + Enable private packages command. + + This command enables picasso private packages by cloning the packages and + defining them as private. + + Raises: + Exception: If there is not enough information to clone the repo. + """ + context = click.get_current_context().obj + tutor_root = context.root + tutor_conf = tutor_config.load(tutor_root) + + private_requirements_path = f"{tutor_root}/env/build/openedx/requirements" + packages = get_picasso_packages(tutor_conf) + + if not os.path.exists(private_requirements_path): + os.makedirs(private_requirements_path) + + for package, info in packages.items(): + if not {"name", "repo", "version"}.issubset(info): + raise click.ClickException( + f"{package} is missing one of the required keys: 'name', 'repo', 'version'" + ) + + if os.path.isdir(f'{private_requirements_path}/{info["name"]}'): + subprocess.call( + ["rm", "-rf", f'{private_requirements_path}/{info["name"]}'] + ) + + subprocess.call( + ["git", "clone", "-b", info["version"], info["repo"]], + cwd=private_requirements_path, + ) + + handle_private_requirements_by_tutor_version(info, private_requirements_path) + + +def handle_private_requirements_by_tutor_version( + info: Dict[str, str], private_requirements_path: str +) -> None: + """ + Handle the private requirements based on the Tutor version. + + Args: + info (Dict[str, str]): A dictionary containing metadata about the requirement. Expected to have a "name" key. + private_requirements_path (str): The directory path to store the private requirements. + """ + tutor_version_obj = Version(tutor_version) + quince_version_obj = Version("v17.0.0") + + if tutor_version_obj < quince_version_obj: + private_txt_path = f"{private_requirements_path}/private.txt" + _enable_private_requirements_before_quince(info, private_txt_path) + else: + _enable_private_requirements_latest(info, private_requirements_path) + + +def _enable_private_requirements_before_quince( + info: Dict[str, str], private_requirements_txt: str +) -> None: + """ + Copy the requirement name in the private.txt file to ensure that requirements are added in the build process for Tutor versions < Quince. + + Args: + info (Dict[str, str]): A dictionary containing metadata about the requirement. Expected to have a "name" key. + private_requirements_txt (str): The file path to `private.txt`, which stores the list of private requirements to be included in the build. + """ + if not os.path.exists(private_requirements_txt): + with open(private_requirements_txt, "w") as file: + file.write("") + + echo_command = f'echo "-e ./{info["name"]}/" >> {private_requirements_txt}' + subprocess.call(echo_command, shell=True) + + +def _enable_private_requirements_latest( + info: Dict[str, str], private_requirements_path: str +) -> None: + """ + Use the tutor mounts method to ensure that requirements are added in the build process. + + Args: + info (Dict[str, str]): A dictionary containing metadata about the requirement. Expected to have a "name" key. + private_requirements_path (str): The root directory where private requirements are stored. + """ + subprocess.call( + [ + "tutor", + "mounts", + "add", + f'{private_requirements_path}/{info["name"]}', + ] + ) + + +def get_picasso_packages(settings: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + """ + Get the distribution packages from the provided settings. + + Args: + settings (dict): The tutor configuration settings. + + Returns: + A dictionary of distribution packages, where the keys are package names + and the values are package details. + """ + picasso_packages = { + key: val for key, val in settings.items() if key.endswith("_DPKG") and val + } + return picasso_packages diff --git a/tutorpicasso/commands/enable_themes.py b/tutorpicasso/commands/enable_themes.py new file mode 100644 index 0000000..a97e24f --- /dev/null +++ b/tutorpicasso/commands/enable_themes.py @@ -0,0 +1,40 @@ +import os +import subprocess +from typing import Any + +import click +from tutor import config as tutor_config + + +@click.command(name="enable-themes", help="Enable picasso themes") +def enable_themes() -> None: + """ + Enable picasso themes. + + This function enables the themes specified in the `PICASSO_THEMES` configuration + and applies them using the ThemeEnabler and ThemeGitRepository classes. + """ + context = click.get_current_context().obj + tutor_root = context.root + tutor_conf = tutor_config.load(tutor_root) + + if not tutor_conf.get("PICASSO_THEMES"): + return + + # We use `type: ignore` for the `tutor_conf` object + # because it comes from the Tutor framework. + # We are not handle type errors related to this object. + for theme in tutor_conf["PICASSO_THEMES"]: # type: ignore + if not {"name", "repo", "version"}.issubset(theme.keys()): # type: ignore + raise click.ClickException( + f"{theme} is missing one or more required keys: " + "'name', 'repo', 'version'" + ) + + theme_path = f'{tutor_root}/env/build/openedx/themes/{theme["name"]}' # type: ignore + if os.path.isdir(theme_path): + subprocess.call(["rm", "-rf", theme_path]) + + subprocess.call( + ["git", "clone", "-b", theme["version"], theme["repo"], theme_path], # type: ignore + ) diff --git a/tutorpicasso/commands/run_extra_commands.py b/tutorpicasso/commands/run_extra_commands.py new file mode 100644 index 0000000..0e2c9ca --- /dev/null +++ b/tutorpicasso/commands/run_extra_commands.py @@ -0,0 +1,143 @@ +import re +import subprocess +from itertools import chain + +from typing import Any, List, Pattern + +import click +from tutor import config as tutor_config + +COMMAND_CHAINING_OPERATORS = ["&&", "&", "||", "|", ";"] + + +@click.command(name="run-extra-commands", help="Run tutor commands") +def run_extra_commands() -> None: + """ + This command runs tutor commands defined in PICASSO_EXTRA_COMMANDS + """ + context = click.get_current_context().obj + tutor_conf = tutor_config.load(context.root) + + picasso_extra_commands: Any = tutor_conf.get("PICASSO_EXTRA_COMMANDS", None) + + if not picasso_extra_commands: + return + + error_message = validate_commands(picasso_extra_commands) + if error_message: + raise click.ClickException(error_message) + + list(map(run_command, picasso_extra_commands)) + + +def validate_commands(commands: Any) -> str: + """ + Takes all the extra commands sent through config.yml and verifies that + all the commands are correct before executing them + + Args: + commands (list[str] | None): The commands sent through PICASSO_EXTRA_COMMANDS in config.yml + """ + splitted_commands = [ + split_string(command, COMMAND_CHAINING_OPERATORS) for command in commands + ] + flat_commands_list: chain[str] = chain.from_iterable(splitted_commands) + + invalid_commands = [] + misspelled_commands = [] + for command in flat_commands_list: + if "tutor" in command.lower(): + continue + if find_tutor_misspelled(command): + misspelled_commands.append(command) + else: + invalid_commands.append(command) + + error_message = "" + if invalid_commands: + error_message += ( + f"Found some issues with the commands:\n\n" + f"=> Invalid commands: {', '.join(invalid_commands)}\n" + ) + + if misspelled_commands: + error_message += f"=> Misspelled commands: {', '.join(misspelled_commands)}\n" + + if error_message: + error_message += ( + "Take a look at the official Tutor commands: " + "https://docs.tutor.edly.io/reference/cli/index.html" + ) + return error_message + + +def run_command(command: str) -> None: + """ + Run an extra command. + + This method runs the extra command provided. + + Args: + command (str): Tutor command. + """ + with subprocess.Popen( + command, + shell=True, + executable="/bin/bash", + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as process: + + stdout, stderr = process.communicate() + + if process.returncode != 0 or "error" in stderr.lower(): + raise click.ClickException( + f"Command '{command}' failed with return code {process.returncode}. Output: {stdout}. Error: {stderr}" + ) + + click.echo(stdout) + + +def find_tutor_misspelled(command: str) -> bool: + """ + Look for misspelled occurrences of the word `tutor` in a given string. E.g. ... + + Args: + command (str): string to be reviewed. + + Return: + True if any misspelled occurrence is found, False otherwise. + """ + return bool(re.match(r"[tT](?:[oru]{3}|[oru]{2}[rR]|[oru]u?)", command)) + + +def create_regex_from_list(special_chars: List[str]) -> Pattern[str]: + """ + Compile a new regex and escape special characters in the given string. + escaping special characters + + Args: + special_chars (list[str]): String that would be used to create a new regex + + Return: + A new compiled regex pattern that can be used for comparisons + """ + escaped_special_chars = list(map(re.escape, special_chars)) # type: ignore + regex_pattern = "|".join(escaped_special_chars) # type: ignore + return re.compile(regex_pattern) + + +def split_string(string: str, split_by: List[str]) -> List[str]: + """ + Split strings based on given patterns. + + Args: + string (str): string to be split + split_by (list[str]): patterns to be used to split the string + + Return: + The string split into a list + """ + return re.split(create_regex_from_list(split_by), string) diff --git a/tutorpicasso/plugin.py b/tutorpicasso/plugin.py index 60be7db..2c06e61 100644 --- a/tutorpicasso/plugin.py +++ b/tutorpicasso/plugin.py @@ -5,7 +5,11 @@ import click import importlib_resources +from packaging.version import Version from tutor import hooks +from tutor.__about__ import __version__ as tutor_version + +from tutorpicasso.commands.cli import picasso from .__about__ import __version__ @@ -13,6 +17,15 @@ # CONFIGURATION ######################################## +# Tutor introduces the MOUNTED_DIRECTORIES in the latest Palm version. +latest_palm_version = "16.1.8" +if Version(tutor_version) > Version(latest_palm_version): + hooks.Filters.MOUNTED_DIRECTORIES.add_items( + [ + ("openedx", r"eox-.*"), + ] + ) + hooks.Filters.CONFIG_DEFAULTS.add_items( [ # Add your new settings that have default values here. @@ -205,6 +218,11 @@ # To define a command group for your plugin, you would define a Click # group and then add it to CLI_COMMANDS: +hooks.Filters.CLI_COMMANDS.add_items( + [ + picasso, + ] +) ### @click.group() ### def picasso() -> None: