Fix Julia sysimage creation

spine_items/tool/tool_instance.py

@@ -29,6 +29,7 @@
     JuliaPersistentExecutionManager,
     PythonPersistentExecutionManager,
 )
+from spine_items.utils import escape_backward_slashes
 
 
 class ToolInstance:
@@ -156,11 +157,11 @@ def prepare(self, args):
         sysimage = self._owner.options.get("julia_sysimage", "")
         use_julia_kernel = self._settings.value("appSettings/useJuliaKernel", defaultValue="2")
         # Prepare args
-        mod_work_dir = repr(self.basedir).strip("'")
+        mod_work_dir = escape_backward_slashes(self.basedir)
         self.args = [f'cd("{mod_work_dir}");']
         cmdline_args = self.tool_specification.cmdline_args + args
         if cmdline_args:
-            fmt_cmdline_args = '["' + repr('", "'.join(cmdline_args)).strip("'") + '"]'
+            fmt_cmdline_args = '["' + escape_backward_slashes('", "'.join(cmdline_args)) + '"]'
             self.args += [f"empty!(ARGS); append!(ARGS, {fmt_cmdline_args});"]
         self.args += [f'include("{self.tool_specification.main_prgm}")']
         if use_julia_kernel == "2":
@@ -185,7 +186,8 @@ def prepare(self, args):
         if use_julia_kernel == "1":
             self.program = julia_exe
             self.args = []
-            self.args.append(f"--project={julia_project_path}")
+            if julia_project_path:
+                self.args.append(f"--project={julia_project_path}")
             if os.path.isfile(sysimage):
                 self.args.append(f"--sysimage={sysimage}")
             if self.tool_specification.main_prgm:
@@ -194,7 +196,8 @@ def prepare(self, args):
             self.exec_mngr = ProcessExecutionManager(self._logger, self.program, *self.args, workdir=self.basedir)
             return
         self.program = [julia_exe]
-        self.program.append(f"--project={julia_project_path}")
+        if julia_project_path:
+            self.program.append(f"--project={julia_project_path}")
         if os.path.isfile(sysimage):
             self.program.append(f"--sysimage={sysimage}")
         alias = f"julia {' '.join([self.tool_specification.main_prgm, *cmdline_args])}"
@@ -268,7 +271,7 @@ def prepare(self, args):
             fp = self.tool_specification.main_prgm
             full_fp = os.path.join(self.basedir, self.tool_specification.main_prgm).replace(os.sep, "/")
             cmdline_args = [full_fp] + self.tool_specification.cmdline_args + args
-            fmt_cmdline_args = '["' + repr('", "'.join(cmdline_args)).strip("'") + '"]'
+            fmt_cmdline_args = '["' + escape_backward_slashes('", "'.join(cmdline_args)) + '"]'
             self.args += [f"import sys; sys.argv = {fmt_cmdline_args};"]
             self.args += [f"import os; os.chdir({repr(self.basedir)})"]
             self.args += self._make_exec_code(fp, full_fp)

spine_items/tool/widgets/options_widgets.py

@@ -20,6 +20,8 @@
 from PySide6.QtCore import Qt, Slot, QVariantAnimation, QPointF
 from PySide6.QtWidgets import QWidget, QFileDialog
 from PySide6.QtGui import QIcon, QLinearGradient, QPalette, QBrush
+
+from spine_items.utils import escape_backward_slashes
 from spinetoolbox.spine_engine_worker import SpineEngineWorker
 from spinetoolbox.execution_managers import QProcessExecutionManager
 from spinetoolbox.helpers import get_open_file_name_in_last_dir, CharIconEngine, make_settings_dict_for_engine
@@ -53,7 +55,6 @@ def _logger(self):
 
 class JuliaOptionsWidget(OptionsWidget):
     def __init__(self):
-        """Init class."""
         from ..ui.julia_options import Ui_Form  # pylint: disable=import-outside-toplevel
 
         super().__init__()
@@ -69,7 +70,7 @@ def __init__(self):
         self.ui.toolButton_open_sysimage.clicked.connect(self._open_sysimage)
         self.ui.toolButton_abort_sysimage.clicked.connect(self._abort_sysimage)
         self.ui.toolButton_abort_sysimage.setVisible(False)
-        self.ui.lineEdit_sysimage.editingFinished.connect(self._handle_le_sysimage_editing_finished)
+        self.ui.lineEdit_sysimage.editingFinished.connect(self._handle_sysimage_editing_finished)
 
     def _make_work_animation(self):
         """
@@ -168,7 +169,7 @@ def do_update_options(self, options):
         self.ui.lineEdit_sysimage.setText(self.last_sysimage_path)
 
     @Slot()
-    def _handle_le_sysimage_editing_finished(self):
+    def _handle_sysimage_editing_finished(self):
         sysimage_path = self.ui.lineEdit_sysimage.text()
         self._tool.update_options({"julia_sysimage": sysimage_path})
 
@@ -223,17 +224,17 @@ def _create_sysimage(self, _checked=False):
         dag = self._project.dag_with_node(self._tool.name)
         if not dag:
             return
-        if dag.nodes():
-            # FIXME?
-            return
         self.sysimage_path = self._get_sysimage_path()
         if self.sysimage_path is None:
             return
         execution_permits = {item_name: item_name == self._tool.name for item_name in dag.nodes}
         settings = make_settings_dict_for_engine(self._settings)
         settings["appSettings/useJuliaKernel"] = "1"  # Use subprocess
         dag_identifier = f"containing {self._tool.name}"
-        self.sysimage_worker = self._project.create_engine_worker(dag, execution_permits, dag_identifier, settings)
+        job_id = self._project.LOCAL_EXECUTION_JOB_ID
+        self.sysimage_worker = self._project.create_engine_worker(
+            dag, execution_permits, dag_identifier, settings, job_id
+        )
         # Use the modified spec
         engine_data = self.sysimage_worker.get_engine_data()
         spec_names = {spec["name"] for type_specs in engine_data["specifications"].values() for spec in type_specs}
@@ -251,7 +252,7 @@ def _create_sysimage(self, _checked=False):
         self._update_ui()
         self.sysimage_worker.start(silent=True)
         self._logger.msg_success.emit(
-            f"Process to create <b>{self.sysimage_basename}</b> sucessfully started.\n"
+            f"Process to create <b>{self.sysimage_basename}</b> successfully started.\n"
             "This process might take a while, but you can keep using Spine Toolbox as normal in the meantime."
         )
 
@@ -278,26 +279,24 @@ def _make_sysimage_spec(self, spec_names):
         precompile_statements_file = self._get_precompile_statements_filepath()
         with open(original_program_file, 'r') as original:
             original_code = original.read()
-        new_code = f"""
-        macro write_loaded_modules(ex)
-            return quote
-                local before = copy(Base.loaded_modules)
-                local val = $(esc(ex))
-                local after = copy(Base.loaded_modules)
-                open("{loaded_modules_file}", "w") do f
-                    print(f, join(setdiff(values(after), values(before)), " "))
-                end
-                val
-            end
-        end
-        @write_loaded_modules begin 
-            {original_code}
+        new_code = f"""macro write_loaded_modules(ex)
+    return quote
+        local before = copy(Base.loaded_modules)
+        local val = $(esc(ex))
+        local after = copy(Base.loaded_modules)
+        open("{escape_backward_slashes(loaded_modules_file)}", "w") do f
+            print(f, join(setdiff(values(after), values(before)), " "))
         end
-        """
+        val
+    end
+end
+@write_loaded_modules begin 
+    {original_code}
+end"""
         spec.includes.insert(0, "")
         spec.cmdline_args += [f"--trace-compile={precompile_statements_file}", "-e", new_code]
         while True:
-            spec.name = str(uuid.uuid4)
+            spec.name = str(uuid.uuid4())
             if spec.name not in spec_names:
                 break
         return spec
@@ -330,33 +329,29 @@ def _do_create_sysimage(self, tool):
         precompile_statements_file = self._get_precompile_statements_filepath()
         with open(loaded_modules_file, 'r') as f:
             modules = f.read()
-        code = f"""
-            using Pkg;
-            project_dir = dirname(Base.active_project());
-            cp(joinpath(project_dir, "Project.toml"), joinpath(project_dir, "Project.backup"); force=true);
-            cp(joinpath(project_dir, "Manifest.toml"), joinpath(project_dir, "Manifest.backup"); force=true);
-            try
-                modules = split("{modules}", " ");
-                Pkg.add(modules);
-                Pkg.add("PackageCompiler");
-                @eval import PackageCompiler
-                Base.invokelatest(
-                    PackageCompiler.create_sysimage,
-                    Symbol.(modules);
-                    sysimage_path="{self.sysimage_path}",
-                    precompile_statements_file="{precompile_statements_file}"
-                )
-            finally
-                cp(joinpath(project_dir, "Project.backup"), joinpath(project_dir, "Project.toml"); force=true);
-                cp(joinpath(project_dir, "Manifest.backup"), joinpath(project_dir, "Manifest.toml"); force=true);
-            end
-        """
+        code = f"""using Pkg;
+project_dir = dirname(Base.active_project());
+cp(joinpath(project_dir, "Project.toml"), joinpath(project_dir, "Project.backup"); force=true);
+cp(joinpath(project_dir, "Manifest.toml"), joinpath(project_dir, "Manifest.backup"); force=true);
+try
+    modules = split("{modules}", " ");
+    Pkg.add(modules);
+    Pkg.add("PackageCompiler");
+    @eval import PackageCompiler
+    Base.invokelatest(
+        PackageCompiler.create_sysimage,
+        Symbol.(modules);
+        sysimage_path="{escape_backward_slashes(self.sysimage_path)}",
+        precompile_statements_file="{escape_backward_slashes(precompile_statements_file)}"
+    )
+finally
+    cp(joinpath(project_dir, "Project.backup"), joinpath(project_dir, "Project.toml"); force=true);
+    cp(joinpath(project_dir, "Manifest.backup"), joinpath(project_dir, "Manifest.toml"); force=true);
+end"""
         julia, *args = get_julia_command(self._settings)
         args += ["-e", code]
-        self.sysimage_worker = QProcessExecutionManager(self._logger, julia, args, silent=True)
-        self.sysimage_worker.execution_finished.connect(
-            lambda ret, tool=tool: self._handle_sysimage_process_finished(ret, tool)
-        )
+        self.sysimage_worker = QProcessExecutionManager(self._logger, julia, args, silent=False)
+        self.sysimage_worker.execution_finished.connect(lambda ret: self._handle_sysimage_process_finished(ret, tool))
         self.sysimage_worker.start_execution(workdir=self._tool.specification().path)
         # Restore the current self._tool
         self._tool = current_tool
@@ -368,7 +363,7 @@ def _handle_sysimage_process_finished(self, ret, tool):
         Args:
             ret (int): The return code of the process, 0 indicates success.
             tool (Tool): The Tool that started the sysimage creation process.
-                It may be different than the Tool currently using the widget.
+                It may be different from the Tool currently using the widget.
         """
         # Replace self._tool while we run this method
         self._tool, current_tool = tool, self._tool
@@ -383,7 +378,7 @@ def _handle_sysimage_process_finished(self, ret, tool):
         else:
             self._tool.update_options({"julia_sysimage": self.sysimage_path})
             self._logger.msg_success.emit(f"<b>{self.sysimage_basename}</b> created successfully.\n")
-        if tool == current_tool:
+        if tool is current_tool:
             self._update_ui()
         # Restore the current self._tool
         self._tool = current_tool

spine_items/utils.py

@@ -180,3 +180,15 @@ def _single_scenario_name_or_none(resources):
             elif name != scenario_name:
                 return None
     return scenario_name
+
+
+def escape_backward_slashes(string):
+    """Escapes Windows directory separators.
+
+    Args:
+        string (str): string to escape
+
+    Returns:
+        str: escaped string
+    """
+    return string.replace("\\", "\\\\")

tests/tool/test_Tool.py

@@ -197,26 +197,29 @@ def test_find_input_files(self):
             ProjectItemResource("Exporter", "file", "first", url="file:///" + url1, metadata={}, filterable=False),
             ProjectItemResource("Exporter", "url", "second", url="file:///" + url2, metadata={}, filterable=False),
             ProjectItemResource("Exporter", "file", "third", url="file:///" + url3, metadata={}, filterable=False),
-            ProjectItemResource("Exporter", "url", "fourth", url="file:///" + url4, metadata={}, filterable=False)
+            ProjectItemResource("Exporter", "url", "fourth", url="file:///" + url4, metadata={}, filterable=False),
         ]
         result = tool._find_input_files(resources)
         expected = {"input1.csv": [expected_urls["url1"], expected_urls["url3"]], "input2.csv": None}
         self.assertEqual(2, len(result))
         self.assertEqual(expected["input2.csv"], result["input2.csv"])
         self.assertTrue(expected_urls["url3"] in result["input1.csv"] or expected_urls["url1"] in result["input1.csv"])
         resources.pop(0)
-        resources.append(ProjectItemResource(
-            "Exporter", "file", "fifth", url="file:///" + url5, metadata={}, filterable=False)
+        resources.append(
+            ProjectItemResource("Exporter", "file", "fifth", url="file:///" + url5, metadata={}, filterable=False)
         )
         result = tool._find_input_files(resources)
         expected = {'input2.csv': [expected_urls["url5"]], 'input1.csv': [expected_urls["url3"]]}
         self.assertEqual(expected, result)
-        resources.append(ProjectItemResource(
-            "Exporter", "file", "sixth", url="file:///" + url6, metadata={}, filterable=False)
+        resources.append(
+            ProjectItemResource("Exporter", "file", "sixth", url="file:///" + url6, metadata={}, filterable=False)
         )
         tool.specification().inputfiles = set(["input2.csv", os.path.join(self._temp_dir.name, "input3.csv")])
         result = tool._find_input_files(resources)
-        expected = {os.path.join(self._temp_dir.name, "input3.csv"): [expected_urls["url6"]], 'input2.csv': [expected_urls["url5"]]}
+        expected = {
+            os.path.join(self._temp_dir.name, "input3.csv"): [expected_urls["url6"]],
+            'input2.csv': [expected_urls["url5"]],
+        }
         self.assertEqual(expected, result)
 
     def _add_tool(self, item_dict=None):