diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 62799f60d..549d4ba2b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -25,11 +25,6 @@ updates: schedule: interval: "weekly" - - package-ecosystem: "npm" - directory: "/packages/notebook-containerizer" - schedule: - interval: "weekly" - - package-ecosystem: "npm" directory: "/packages/notebook-search" schedule: diff --git a/Makefile b/Makefile index bfb28abef..190b72162 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,6 @@ install-backend: build-backend build-frontend: jlpm-install npx lerna run build --scope @jupyter_vre/chart-customs npx lerna run build --scope @jupyter_vre/core - npx lerna run build --scope @jupyter_vre/notebook-containerizer npx lerna run build --scope @jupyter_vre/notebook-search npx lerna run build --scope @jupyter_vre/dataset-search npx lerna run build --scope @jupyter_vre/components @@ -59,7 +58,6 @@ install-ui: $(call INSTALL_LAB_EXTENSION,notebook-search) $(call INSTALL_LAB_EXTENSION,dataset-search) $(call INSTALL_LAB_EXTENSION,core) - $(call INSTALL_LAB_EXTENSION,notebook-containerizer) $(call INSTALL_LAB_EXTENSION,chart-customs) $(call INSTALL_LAB_EXTENSION,components) $(call INSTALL_LAB_EXTENSION,experiment-manager) @@ -70,7 +68,6 @@ link-ui: $(call LINK_LAB_EXTENSION,notebook-search) $(call LINK_LAB_EXTENSION,dataset-search) $(call LINK_LAB_EXTENSION,core) - $(call LINK_LAB_EXTENSION,notebook-containerizer) $(call LINK_LAB_EXTENSION,chart-customs) $(call LINK_LAB_EXTENSION,components) $(call LINK_LAB_EXTENSION,experiment-manager) @@ -80,7 +77,6 @@ link-ui: dist-ui: build-frontend mkdir -p dist $(call PACKAGE_LAB_EXTENSION,core) - $(call PACKAGE_LAB_EXTENSION,notebook-containerizer) $(call PACKAGE_LAB_EXTENSION,chart-customs) $(call PACKAGE_LAB_EXTENSION,components) $(call PACKAGE_LAB_EXTENSION,experiment-manager) diff --git a/docker/vanilla/dev.Dockerfile b/docker/vanilla/dev.Dockerfile index 81d13818c..02b15ff26 100644 --- a/docker/vanilla/dev.Dockerfile +++ b/docker/vanilla/dev.Dockerfile @@ -41,7 +41,6 @@ COPY packages/chart-customs/package.json packages/chart-customs/ COPY packages/components/package.json packages/components/ COPY packages/core/package.json packages/core/ COPY packages/experiment-manager/package.json packages/experiment-manager/ -COPY packages/notebook-containerizer/package.json packages/notebook-containerizer/ COPY packages/notebook-search/package.json packages/notebook-search/ COPY packages/vre-menu/package.json packages/vre-menu/ COPY packages/vre-panel/package.json packages/vre-panel/ @@ -69,11 +68,11 @@ RUN jupyter serverextension enable --py jupyterlab_vre --user WORKDIR /live/ts COPY --chown=$NB_USER:users packages/ packages/ COPY --chown=$NB_USER:users tsconfig-base.json . -RUN extensions="chart-customs core notebook-containerizer notebook-search components experiment-manager vre-panel vre-menu"; \ +RUN extensions="chart-customs core notebook-search components experiment-manager vre-panel vre-menu"; \ for ext in $extensions; do \ npx lerna run build --scope "@jupyter_vre/$ext"; \ done -RUN extensions="chart-customs core notebook-containerizer notebook-search components experiment-manager vre-panel vre-menu"; \ +RUN extensions="chart-customs core notebook-search components experiment-manager vre-panel vre-menu"; \ for ext in $extensions; do \ jupyter labextension link --no-build "packages/$ext"; \ done diff --git a/jupyterlab_vre/__init__.py b/jupyterlab_vre/__init__.py index 0a748839e..5b5259af9 100644 --- a/jupyterlab_vre/__init__.py +++ b/jupyterlab_vre/__init__.py @@ -7,7 +7,6 @@ from .dataset_search.handlers import DatasetSearchHandler, DatasetDownloadHandler from .experiment_manager.handlers import ExportWorkflowHandler, ExecuteWorkflowHandler from .handlers import CatalogGetAllHandler -from .notebook_containerizer.handlers import NotebookExtractorHandler from .notebook_search.handlers import NotebookSearchHandler, NotebookSearchRatingHandler, NotebookDownloadHandler, \ NotebookSeachHistoryHandler, NotebookSourceHandler from .registries.handlers import RegistriesHandler @@ -34,8 +33,6 @@ def load_jupyter_server_extension(lab_app): ( url_path_join(lab_app.web_app.settings['base_url'], r'/vre/notebooksearchrating'), NotebookSearchRatingHandler), (url_path_join(lab_app.web_app.settings['base_url'], r'/vre/containerizer/extract'), ExtractorHandler), - ( - url_path_join(lab_app.web_app.settings['base_url'], r'/vre/nbcontainerizer/extract'), NotebookExtractorHandler), (url_path_join(lab_app.web_app.settings['base_url'], r'/vre/containerizer/types'), TypesHandler), (url_path_join(lab_app.web_app.settings['base_url'], r'/vre/containerizer/baseimage'), BaseImageHandler), (url_path_join(lab_app.web_app.settings['base_url'], r'/vre/containerizer/addcell'), CellsHandler), diff --git a/jupyterlab_vre/component_containerizer/handlers.py b/jupyterlab_vre/component_containerizer/handlers.py index 1a7659aab..fd3936207 100644 --- a/jupyterlab_vre/component_containerizer/handlers.py +++ b/jupyterlab_vre/component_containerizer/handlers.py @@ -29,9 +29,10 @@ from jupyterlab_vre.database.cell import Cell from jupyterlab_vre.services.containerizer.Rcontainerizer import Rcontainerizer from jupyterlab_vre.services.converter.converter import ConverterReactFlowChart -from jupyterlab_vre.services.extractor.headerextractor import HeaderExtractor +from jupyterlab_vre.services.extractor.extractor import DummyExtractor from jupyterlab_vre.services.extractor.pyextractor import PyExtractor from jupyterlab_vre.services.extractor.rextractor import RExtractor +from jupyterlab_vre.services.extractor.headerextractor import HeaderExtractor logger = logging.getLogger(__name__) @@ -124,27 +125,33 @@ async def post(self, *args, **kwargs): source = notebook.cells[cell_index].source - # extractor based on the cell header - try: - extractor = HeaderExtractor(notebook, source) - except jsonschema.ValidationError as e: - self.set_status(400, f"Invalid cell header") - self.write( - { - 'message': f"Error in cell header: {e}", - 'reason': None, - 'traceback': traceback.format_exception(e), - } - ) - self.flush() - return + if notebook.cells[cell_index].cell_type != 'code': + # dummy extractor for non-code cells (e.g. markdown) + extractor = DummyExtractor(notebook, source) + else: + # extractor based on the cell header + try: + extractor = HeaderExtractor(notebook, source) + except jsonschema.ValidationError as e: + self.set_status(400, f"Invalid cell header") + self.write( + { + 'message': f"Error in cell header: {e}", + 'reason': None, + 'traceback': traceback.format_exception(e), + } + ) + self.flush() + return - # extractor based on the kernel (if cell header is not defined) - if not extractor.enabled(): - if kernel == "IRkernel": - extractor = RExtractor(notebook) - else: - extractor = PyExtractor(notebook) + # Extractor based on code analysis. Used if the cell has no header, + # or if some values are not specified in the header + if not extractor.is_complete(): + if kernel == "IRkernel": + code_extractor = RExtractor(notebook, source) + else: + code_extractor = PyExtractor(notebook, source) + extractor.add_missing_values(code_extractor) extracted_nb = extract_cell_by_index(notebook, cell_index) if kernel == "IRkernel": @@ -159,29 +166,13 @@ async def post(self, *args, **kwargs): if 'JUPYTERHUB_USER' in os.environ: title += '-' + slugify(os.environ['JUPYTERHUB_USER']) - ins = {} - outs = {} - params = {} - confs = [] - dependencies = [] - - # Check if cell is code. If cell is for example markdown we get execution from 'extractor.infer_cell_inputs( - # source)' - if notebook.cells[cell_index].cell_type == 'code': - ins = extractor.infer_cell_inputs(source) - outs = extractor.infer_cell_outputs(source) - - confs = extractor.extract_cell_conf_ref(source) - dependencies = extractor.infer_cell_dependencies(source, confs) - # If any of these change, we create a new cell in the catalog. # This matches the cell properties saved in workflows. cell_identity_dict = { 'title': title, - 'params': params, - 'inputs': ins, - 'outputs': outs, - 'deps': sorted(dependencies, key=lambda x: x['name']), + 'params': extractor.params, + 'inputs': extractor.ins, + 'outputs': extractor.outs, } cell_identity_str = json.dumps(cell_identity_dict, sort_keys=True) node_id = hashlib.sha1(cell_identity_str.encode()).hexdigest()[:7] @@ -191,27 +182,26 @@ async def post(self, *args, **kwargs): title=title, task_name=slugify(title.lower()), original_source=source, - inputs=ins, - outputs=outs, - params=params, - confs=confs, - dependencies=dependencies, + inputs=extractor.ins, + outputs=extractor.outs, + params={}, + confs=extractor.confs, + dependencies=extractor.dependencies, container_source="", kernel=kernel, notebook_dict=extracted_nb.dict() ) - if notebook.cells[cell_index].cell_type == 'code': - cell.integrate_configuration() - params = extractor.extract_cell_params(cell.original_source) - cell.add_params(params) - cell.add_param_values(params) + cell.integrate_configuration() + extractor.params = extractor.extract_cell_params(cell.original_source) + cell.add_params(extractor.params) + cell.add_param_values(extractor.params) node = ConverterReactFlowChart.get_node( node_id, title, - set(ins), - set(outs), - params, + set(extractor.ins), + set(extractor.outs), + extractor.params, ) chart = { @@ -300,6 +290,7 @@ def wait_for_github_api_resources(): sleep(remaining_time + 1) rate_limit = github.get_rate_limit() + def find_job( wf_id=None, wf_creation_utc=None, @@ -340,6 +331,7 @@ def find_job( return job return None + def wait_for_job( wf_id=None, wf_creation_utc=None, @@ -381,12 +373,14 @@ def wait_for_job( return job sleep(5) + def write_cell_to_file(current_cell): Path('/tmp/workflow_cells/cells').mkdir(parents=True, exist_ok=True) with open('/tmp/workflow_cells/cells/' + current_cell.task_name + '.json', 'w') as f: f.write(current_cell.toJSON()) f.close() + class CellsHandler(APIHandler, Catalog): logger = logging.getLogger(__name__) diff --git a/jupyterlab_vre/notebook_containerizer/__init__.py b/jupyterlab_vre/notebook_containerizer/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/jupyterlab_vre/notebook_containerizer/handlers.py b/jupyterlab_vre/notebook_containerizer/handlers.py deleted file mode 100644 index b67b8ff1d..000000000 --- a/jupyterlab_vre/notebook_containerizer/handlers.py +++ /dev/null @@ -1,102 +0,0 @@ -import copy -import json -import logging -import os -import uuid - -import nbformat as nb -from notebook.base.handlers import APIHandler -from slugify import slugify -from tornado import web - -from jupyterlab_vre.database.catalog import Catalog -from jupyterlab_vre.database.cell import Cell -from jupyterlab_vre.services.converter.converter import ConverterReactFlowChart -from jupyterlab_vre.services.extractor.pyextractor import PyExtractor -from jupyterlab_vre.services.extractor.rextractor import RExtractor - - -# TODO: we might have to do something similar here where we have to determine the kernel and based on that get the extractor - -class NotebookExtractorHandler(APIHandler, Catalog): - - @web.authenticated - async def get(self): - msg_json = dict(title='Operation not supported.') - self.write(msg_json) - self.flush() - - @web.authenticated - async def post(self, *args, **kwargs): - - payload = self.get_json_body() - logging.getLogger(__name__).debug('NotebookExtractorHandler. payload: ' + json.dumps(payload, indent=4)) - print('----------------------------------------------') - print('NotebookExtractorHandler. payload: ' + json.dumps(payload, indent=4)) - print('----------------------------------------------') - notebook = nb.reads(json.dumps(payload['notebook']), nb.NO_CONVERT) - kernel = payload['kernel'] - if kernel == "IRkernel": - extractor = RExtractor(notebook) - else: - extractor = PyExtractor(notebook) - source = '' - params = set() - confs = set() - ins = dict() - outs = extractor.infer_cell_outputs(notebook.cells[len(notebook.cells) - 1].source) - title = '' - for cell_source in extractor.sources: - p = extractor.extract_cell_params(cell_source) - params.update(p) - c = extractor.extract_cell_conf_ref(source) - confs.update(c) - source += cell_source + '\n' - - if not title: - title = cell_source.partition('\n')[0].strip() - title = 'notebook-' + slugify(title) - else: - title = 'Untitled' - if 'JUPYTERHUB_USER' in os.environ: - title += '-' + slugify(os.environ['JUPYTERHUB_USER']) - dependencies = extractor.infer_cell_dependencies(source, confs) - - node_id = str(uuid.uuid4())[:7] - cell = Cell( - node_id=node_id, - title=title, - task_name=slugify(title.lower()), - original_source=source, - inputs=ins, - outputs=outs, - params=list(params), - confs=list(confs), - dependencies=list(dependencies), - container_source='' - ) - cell.integrate_configuration() - node = ConverterReactFlowChart.get_node( - node_id, - title, - set(ins), - set(outs), - params, - ) - - chart = { - 'offset': { - 'x': 0, - 'y': 0, - }, - 'scale': 1, - 'nodes': {node_id: node}, - 'links': {}, - 'selected': {}, - 'hovered': {}, - } - cell.chart_obj = chart - Catalog.editor_buffer = copy.deepcopy(cell) - logging.getLogger(__name__).debug('NotebookExtractorHandler. cell: ' + str(cell.toJSON())) - self.write(cell.toJSON()) - self.flush() diff --git a/jupyterlab_vre/services/extractor/extractor.py b/jupyterlab_vre/services/extractor/extractor.py new file mode 100644 index 000000000..dcd3fb867 --- /dev/null +++ b/jupyterlab_vre/services/extractor/extractor.py @@ -0,0 +1,56 @@ +import abc + + +class Extractor(abc.ABC): + ins: dict + outs: dict + params: dict + confs: list + dependencies: list + + def __init__(self, notebook, cell_source): + self.notebook = notebook + self.cell_source = cell_source + + self.ins = self.infer_cell_inputs() + self.outs = self.infer_cell_outputs() + self.params = self.extract_cell_params(cell_source) + self.confs = self.extract_cell_conf_ref() + self.dependencies = self.infer_cell_dependencies(self.confs) + + @abc.abstractmethod + def infer_cell_inputs(self): + pass + + @abc.abstractmethod + def infer_cell_outputs(self): + pass + + @abc.abstractmethod + def extract_cell_params(self, source): + pass + + @abc.abstractmethod + def extract_cell_conf_ref(self): + pass + + @abc.abstractmethod + def infer_cell_dependencies(self, confs): + pass + + +class DummyExtractor(Extractor): + def infer_cell_inputs(self): + return {} + + def infer_cell_outputs(self): + return {} + + def extract_cell_params(self, source): + return {} + + def extract_cell_conf_ref(self): + return [] + + def infer_cell_dependencies(self, confs): + return [] diff --git a/jupyterlab_vre/services/extractor/headerextractor.py b/jupyterlab_vre/services/extractor/headerextractor.py index 5d215bba3..38429c800 100644 --- a/jupyterlab_vre/services/extractor/headerextractor.py +++ b/jupyterlab_vre/services/extractor/headerextractor.py @@ -7,8 +7,10 @@ import jsonschema import yaml +from .extractor import Extractor -class HeaderExtractor: + +class HeaderExtractor(Extractor): """ Extracts cells using information defined by the user in its header Cells should contain a comment with a yaml block defining inputs, outputs, @@ -39,6 +41,11 @@ class HeaderExtractor: The document is validated with the schema `cell_header.schema.json` """ + ins: Union[dict, None] + outs: Union[dict, None] + params: Union[dict, None] + confs: Union[list, None] + dependencies: Union[list, None] def __init__(self, notebook, cell_source): self.re_yaml_doc_in_comment = re.compile( @@ -49,10 +56,10 @@ def __init__(self, notebook, cell_source): ), re.MULTILINE) self.schema = self._load_schema() - - self.notebook = notebook - self.cell_source = cell_source self.cell_header = self._extract_header(cell_source) + self._external_extract_cell_params = None + + super().__init__(notebook, cell_source) @staticmethod def _load_schema(): @@ -66,6 +73,15 @@ def _load_schema(): def enabled(self): return self.cell_header is not None + def is_complete(self): + return ( + (self.ins is not None) + and (self.outs is not None) + and (self.params is not None) + and (self.confs is not None) + and (self.dependencies is not None) + ) + def _extract_header(self, cell_source): # get yaml document from cell comments m = self.re_yaml_doc_in_comment.match(cell_source) @@ -87,6 +103,25 @@ def _extract_header(self, cell_source): raise e return header + def add_missing_values(self, extractor: Extractor): + """ Add values not specified in the header from another extractor + (e.g. PyExtractor or RExtractor) + """ + if self.ins is None: + self.ins = extractor.ins + if self.outs is None: + self.outs = extractor.outs + if self.params is None: + self.params = extractor.params + # We store a reference to extractor.extract_cell_params because + # self.extract_cell_params is called after self.add_missing_values + # in component_containerizer.handlers.ExtractorHandler.post() + self._external_extract_cell_params = extractor.extract_cell_params + if self.confs is None: + self.confs = extractor.confs + if self.dependencies is None: + self.dependencies = extractor.dependencies + @staticmethod def _parse_inputs_outputs_param_items( item: Union[str, dict], @@ -155,32 +190,52 @@ def _parse_inputs_outputs_param_items( def _infer_cell_inputs_outputs_params( self, - source, + header: Union[dict, None], item_type: Literal['inputs', 'outputs', 'params'], - ) -> dict: - header = self._extract_header(source) - items = header['NaaVRE']['cell'].get(item_type, []) + ) -> Union[dict, None]: + if header is None: + return None + items = header['NaaVRE']['cell'].get(item_type) + if items is None: + return None items = [self._parse_inputs_outputs_param_items(it, item_type) for it in items] return {it['name']: it for it in items} - def infer_cell_inputs(self, source): - return self._infer_cell_inputs_outputs_params(source, 'inputs') + def infer_cell_inputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'inputs', + ) - def infer_cell_outputs(self, source): - return self._infer_cell_inputs_outputs_params(source, 'outputs') + def infer_cell_outputs(self): + return self._infer_cell_inputs_outputs_params( + self.cell_header, + 'outputs', + ) def extract_cell_params(self, source): - return self._infer_cell_inputs_outputs_params(source, 'params') - - def extract_cell_conf_ref(self, source): - header = self._extract_header(source) - items = header['NaaVRE']['cell'].get('confs', []) + if self._external_extract_cell_params is not None: + return self._external_extract_cell_params(source) + return self._infer_cell_inputs_outputs_params( + self._extract_header(source), + 'params', + ) + + def extract_cell_conf_ref(self): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('confs') + if items is None: + return None return {k: v['assignation'] for it in items for k, v in it.items()} - def infer_cell_dependencies(self, source, confs): - header = self._extract_header(source) - items = header['NaaVRE']['cell'].get('dependencies', []) + def infer_cell_dependencies(self, confs): + if self.cell_header is None: + return None + items = self.cell_header['NaaVRE']['cell'].get('dependencies') + if items is None: + return None return [ { 'name': it.get('name'), diff --git a/jupyterlab_vre/services/extractor/pyextractor.py b/jupyterlab_vre/services/extractor/pyextractor.py index b8969e70f..5671b3d2b 100644 --- a/jupyterlab_vre/services/extractor/pyextractor.py +++ b/jupyterlab_vre/services/extractor/pyextractor.py @@ -7,15 +7,17 @@ from pytype import config as pytype_config from pytype.tools.annotate_ast import annotate_ast +from .extractor import Extractor -class PyExtractor: + +class PyExtractor(Extractor): sources: list imports: dict configurations: dict global_params: dict undefined: dict - def __init__(self, notebook): + def __init__(self, notebook, cell_source): # If cell_type is code and not starting with '!' self.sources = [nbcell.source for nbcell in notebook.cells if nbcell.cell_type == 'code' and len(nbcell.source) > 0 and nbcell.source[0] != '!'] @@ -30,6 +32,8 @@ def __init__(self, notebook): for source in self.sources: self.undefined.update(self.__extract_cell_undefined(source)) + super().__init__(notebook, cell_source) + def __extract_imports(self, sources): imports = {} for s in sources: @@ -91,20 +95,20 @@ def __extract_params(self, sources): } return params - def infer_cell_outputs(self, cell_source): - cell_names = self.__extract_cell_names(cell_source) + def infer_cell_outputs(self): + cell_names = self.__extract_cell_names(self.cell_source) return { name: properties for name, properties in cell_names.items() - if name not in self.__extract_cell_undefined(cell_source) + if name not in self.__extract_cell_undefined(self.cell_source) and name not in self.imports and name in self.undefined and name not in self.configurations and name not in self.global_params } - def infer_cell_inputs(self, cell_source): - cell_undefined = self.__extract_cell_undefined(cell_source) + def infer_cell_inputs(self): + cell_undefined = self.__extract_cell_undefined(self.cell_source) return { und: properties for und, properties in cell_undefined.items() @@ -113,9 +117,9 @@ def infer_cell_inputs(self, cell_source): and und not in self.global_params } - def infer_cell_dependencies(self, cell_source, confs): + def infer_cell_dependencies(self, confs): dependencies = [] - names = self.__extract_cell_names(cell_source) + names = self.__extract_cell_names(self.cell_source) for ck in confs: names.update(self.__extract_cell_names(confs[ck])) @@ -235,9 +239,9 @@ def extract_cell_params(self, cell_source): params[u] = self.global_params[u] return params - def extract_cell_conf_ref(self, cell_source): + def extract_cell_conf_ref(self): confs = {} - cell_unds = self.__extract_cell_undefined(cell_source) + cell_unds = self.__extract_cell_undefined(self.cell_source) conf_unds = [und for und in cell_unds if und in self.configurations] for u in conf_unds: if u not in confs: diff --git a/jupyterlab_vre/services/extractor/rextractor.py b/jupyterlab_vre/services/extractor/rextractor.py index 027d932d4..228074c3f 100644 --- a/jupyterlab_vre/services/extractor/rextractor.py +++ b/jupyterlab_vre/services/extractor/rextractor.py @@ -7,6 +7,9 @@ import rpy2.robjects.packages as rpackages from rpy2.robjects.packages import importr +from .extractor import Extractor + + # Create an R environment r_env = robjects.globalenv @@ -99,14 +102,14 @@ # TODO: create an interface such that it can be easily extended to other kernels -class RExtractor: +class RExtractor(Extractor): sources: list imports: dict configurations: dict global_params: dict undefined: dict - def __init__(self, notebook): + def __init__(self, notebook, cell_source): self.sources = [nbcell.source for nbcell in notebook.cells if nbcell.cell_type == 'code' and len(nbcell.source) > 0] @@ -117,6 +120,8 @@ def __init__(self, notebook): for source in self.sources: self.undefined.update(self.__extract_cell_undefined(source)) + super().__init__(notebook, cell_source) + def __extract_imports(self, sources): imports = {} for s in sources: @@ -213,20 +218,20 @@ def __extract_params(self, sources): # check source https://adv-r.hadley.nz/exp } return params - def infer_cell_outputs(self, cell_source): - cell_names = self.__extract_cell_names(cell_source) + def infer_cell_outputs(self): + cell_names = self.__extract_cell_names(self.cell_source) return { name: properties for name, properties in cell_names.items() - if name not in self.__extract_cell_undefined(cell_source) + if name not in self.__extract_cell_undefined(self.cell_source) and name not in self.imports and name in self.undefined and name not in self.configurations and name not in self.global_params } - def infer_cell_inputs(self, cell_source): - cell_undefined = self.__extract_cell_undefined(cell_source) + def infer_cell_inputs(self): + cell_undefined = self.__extract_cell_undefined(self.cell_source) return { und: properties for und, properties in cell_undefined.items() @@ -235,7 +240,7 @@ def infer_cell_inputs(self, cell_source): and und not in self.global_params } - def infer_cell_dependencies(self, cell_source, confs): + def infer_cell_dependencies(self, confs): # TODO: check this code, you have removed logic. # we probably like to only use dependencies that are necessary to execute the cell # however this is challenging in R as functions are non-scoped @@ -381,9 +386,9 @@ def extract_cell_params(self, cell_source): params[u] = self.global_params[u] return params - def extract_cell_conf_ref(self, cell_source): + def extract_cell_conf_ref(self): confs = {} - cell_unds = self.__extract_cell_undefined(cell_source) + cell_unds = self.__extract_cell_undefined(self.cell_source) conf_unds = [und for und in cell_unds if und in self.configurations] for u in conf_unds: if u not in confs: diff --git a/jupyterlab_vre/tests/test_extractor.py b/jupyterlab_vre/tests/test_extractor.py index c7e0cf306..22658a41f 100644 --- a/jupyterlab_vre/tests/test_extractor.py +++ b/jupyterlab_vre/tests/test_extractor.py @@ -28,12 +28,12 @@ def create_cell(payload_path=None): cell_index = payload['cell_index'] notebook = nb.reads(json.dumps(payload['notebook']), nb.NO_CONVERT) + source = notebook.cells[cell_index].source if payload['kernel'] == "IRkernel": - extractor = RExtractor(notebook) + extractor = RExtractor(notebook, source) else: - extractor = PyExtractor(notebook) + extractor = PyExtractor(notebook, source) - source = notebook.cells[cell_index].source title = source.partition('\n')[0] title = slugify(title) if title and title[ 0] == "#" else "Untitled" @@ -50,11 +50,11 @@ def create_cell(payload_path=None): # Check if cell is code. If cell is for example markdown we get execution from 'extractor.infere_cell_inputs( # source)' if notebook.cells[cell_index].cell_type == 'code': - ins = extractor.infer_cell_inputs(source) - outs = extractor.infer_cell_outputs(source) + ins = extractor.infer_cell_inputs() + outs = extractor.infer_cell_outputs() - confs = extractor.extract_cell_conf_ref(source) - dependencies = extractor.infer_cell_dependencies(source, confs) + confs = extractor.extract_cell_conf_ref() + dependencies = extractor.infer_cell_dependencies(confs) node_id = str(uuid.uuid4())[:7] cell = Cell( diff --git a/packages/notebook-containerizer/package.json b/packages/notebook-containerizer/package.json deleted file mode 100644 index b99c23816..000000000 --- a/packages/notebook-containerizer/package.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "@jupyter_vre/notebook-containerizer", - "version": "1.0.0", - "description": "NaaVRE Notebook Containerizer", - "keywords": [ - "jupyter", - "jupyterlab", - "jupyterlab-extension" - ], - "homepage": "https://github.com/QCDIS/NaaVRE", - "bugs": { - "url": "https://github.com/QCDIS/NaaVRE/issues" - }, - "license": "BSD-3-Clause", - "author": "Riccardo Bianchi", - "files": [ - "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}", - "src/**/*.{ts,tsx}", - "style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}" - ], - "main": "lib/index.js", - "types": "lib/index.d.ts", - "style": "style/index.css", - "repository": { - "type": "git", - "url": "https://github.com/QCDIS/NaaVRE.git" - }, - "scripts": { - "build": "tsc", - "dist": "yarn pack", - "clean": "rimraf lib", - "watch": "tsc -w" - }, - "dependencies": { - "@jupyterlab/notebook": "3.5.3" - }, - "devDependencies": { - "@types/node": "20.2.5", - "rimraf": "5.0.0", - "typescript": "5.0.2" - }, - "publishConfig": { - "access": "public" - }, - "jupyterlab": { - "extension": true - } -} \ No newline at end of file diff --git a/packages/notebook-containerizer/src/NotebookContainerizerDialog.tsx b/packages/notebook-containerizer/src/NotebookContainerizerDialog.tsx deleted file mode 100644 index cd4e4abf3..000000000 --- a/packages/notebook-containerizer/src/NotebookContainerizerDialog.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import { INotebookModel, INotebookTracker } from '@jupyterlab/notebook'; -import { styled, ThemeProvider } from '@material-ui/core'; -import * as React from 'react'; -import { theme } from './Theme'; -import { requestAPI } from '@jupyter_vre/core'; - -interface IState { -} - -export const DefaultState: IState = { -} - -const CatalogBody = styled('div')({ - display: 'flex', - overflow: 'hidden', - flexDirection: 'row', -}) - -interface NotebookContainerizerDialogProps { - - notebookTracker: INotebookTracker -} - -export class NotebookContainerizerDialog extends React.Component { - - state = DefaultState - - constructor(props: NotebookContainerizerDialogProps) { - super(props); - } - - exctractor = async (notebookModel: INotebookModel, _save = false) => { - try { - const resp = await requestAPI('nbcontainerizer/extract', { - body: JSON.stringify({ - notebook: notebookModel.toJSON() - }), - method: 'POST' - }); - - console.log(resp); - - } catch (error) { - console.log(error); - } - } - - componentDidMount(): void { - - this.exctractor(this.props.notebookTracker.currentWidget.model); - } - - render(): React.ReactElement { - return ( - -

Containerize Notebook

- - -
- ) - } -} \ No newline at end of file diff --git a/packages/notebook-containerizer/src/Theme.ts b/packages/notebook-containerizer/src/Theme.ts deleted file mode 100644 index 5a98d6fdf..000000000 --- a/packages/notebook-containerizer/src/Theme.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { createTheme } from '@material-ui/core'; - -export const theme = createTheme({ - palette: { - secondary: { - main: '#ea5b2d', - dark: '#b12800', - light: '#ff8c5a', - }, - primary: { - main: '#0f4e8a', - dark: '#00275c', - light: '#4e79ba', - }, - }, -}); diff --git a/packages/notebook-containerizer/src/index.tsx b/packages/notebook-containerizer/src/index.tsx deleted file mode 100644 index 3e36ebacc..000000000 --- a/packages/notebook-containerizer/src/index.tsx +++ /dev/null @@ -1,77 +0,0 @@ -import { - JupyterFrontEnd, - JupyterFrontEndPlugin -} from '@jupyterlab/application'; -import { ToolbarButton, ReactWidget, Dialog, showDialog } from '@jupyterlab/apputils'; -import { DocumentRegistry } from '@jupyterlab/docregistry'; -import { - INotebookModel, INotebookTracker, NotebookPanel -} from '@jupyterlab/notebook'; -import { DisposableDelegate, IDisposable } from '@lumino/disposable'; -import * as React from 'react'; -import { NotebookContainerizerDialog } from './NotebookContainerizerDialog'; - -/** - * The plugin registration information. - */ -const plugin: JupyterFrontEndPlugin = { - activate, - id: 'toolbar-containerize-notebook', - autoStart: true, - requires: [INotebookTracker] -}; - -export class NotebookSearchExtension implements DocumentRegistry.IWidgetExtension { - - notebookTracker: INotebookTracker - - constructor(notebookTracker: INotebookTracker) { - this.notebookTracker = notebookTracker; - } - - createNew( - panel: NotebookPanel, - _context: DocumentRegistry.IContext - ): IDisposable { - - const containerizeNotebook = () => { - - const catalogOptions: Partial> = { - title: '', - body: ReactWidget.create( - - ) as Dialog.IBodyWidget, - buttons: [] - }; - - - showDialog(catalogOptions); - }; - - const button = new ToolbarButton({ - className: 'notebook-containerizer', - label: 'Notebooks Containerizer', - onClick: containerizeNotebook, - tooltip: 'Notebooks Containerizer', - }); - - panel.toolbar.insertItem(10, 'containerizeNotebooks', button); - return new DisposableDelegate(() => { - button.dispose(); - }); - } -} - -/** - * Activate the extension. - * - * @param app Main application object - */ -function activate(app: JupyterFrontEnd, notebookTracker: INotebookTracker): void { - app.docRegistry.addWidgetExtension('Notebook', new NotebookSearchExtension(notebookTracker)); -} - -/** - * Export the plugin as default. - */ -export default plugin; \ No newline at end of file diff --git a/packages/notebook-containerizer/style/index.css b/packages/notebook-containerizer/style/index.css deleted file mode 100644 index 6e02bdfae..000000000 --- a/packages/notebook-containerizer/style/index.css +++ /dev/null @@ -1 +0,0 @@ -/* Fill */ \ No newline at end of file diff --git a/packages/notebook-containerizer/tsconfig.json b/packages/notebook-containerizer/tsconfig.json deleted file mode 100644 index 5df522e9b..000000000 --- a/packages/notebook-containerizer/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../../tsconfig-base", - "compilerOptions": { - "module": "commonjs", - "outDir": "lib", - "rootDir": "src", - "target": "es6", - "types": ["node"], - "typeRoots": ["./node_modules/@types"], - "skipLibCheck": true - }, - "include": ["src/*", "src/*/*"] -} \ No newline at end of file