diff --git a/computation-slurm-test/src/test/java/com/powsybl/computation/slurm/SlurmNormalExecutionTest.java b/computation-slurm-test/src/test/java/com/powsybl/computation/slurm/SlurmNormalExecutionTest.java index dec17fc..a15ee1b 100644 --- a/computation-slurm-test/src/test/java/com/powsybl/computation/slurm/SlurmNormalExecutionTest.java +++ b/computation-slurm-test/src/test/java/com/powsybl/computation/slurm/SlurmNormalExecutionTest.java @@ -120,6 +120,31 @@ public String after(Path workingDir, ExecutionReport report) throws IOException baseTest(supplier); } + @Test + public void testFilesWithSpaces() { + Supplier> supplier = () -> new AbstractExecutionHandler() { + @Override + public List before(Path workingDir) { + generateZipFileOnRemote("in 0", workingDir.resolve("in 0.zip")); + generateZipFileOnRemote("in 1", workingDir.resolve("in 1.zip")); + generateZipFileOnRemote("in 2", workingDir.resolve("in 2.zip")); + return CommandExecutionsTestFactory.testFilesWithSpaces(3); + } + + @Override + public String after(Path workingDir, ExecutionReport report) throws IOException { + super.after(workingDir, report); + Path out2 = workingDir.resolve("out 2.gz"); + System.out.println("out 2.gz should exists, actual exists:" + Files.exists(out2)); + if (Files.exists(out2)) { + return "OK"; + } + return "KO"; + } + }; + baseTest(supplier); + } + @Test public void testGroupCmd() { Supplier> supplier = () -> new AbstractReturnOKExecutionHandler() { diff --git a/computation-slurm/src/main/java/com/powsybl/computation/slurm/SbatchScriptGenerator.java b/computation-slurm/src/main/java/com/powsybl/computation/slurm/SbatchScriptGenerator.java index 8549108..edca4d9 100644 --- a/computation-slurm/src/main/java/com/powsybl/computation/slurm/SbatchScriptGenerator.java +++ b/computation-slurm/src/main/java/com/powsybl/computation/slurm/SbatchScriptGenerator.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; /** * Generates the content of the sbatch script, to be submitted using and {@link SbatchCmd}. @@ -42,6 +43,9 @@ class SbatchScriptGenerator { private static final String PRE_FILE = "PRE"; private static final String POST_FILE = "POST"; + private static final Function WRAP_FILENAME = str -> "'" + str + "'"; + private static final Function VAR2ARG = str -> "\"$" + str + "\""; + private final Path flagDir; SbatchScriptGenerator(Path flagDir) { @@ -54,7 +58,7 @@ List unzipCommonInputFiles(Command command) { command.getInputFiles() .stream().filter(inputFile -> !inputFile.dependsOnExecutionNumber()) // common .filter(inputFile -> inputFile.getPreProcessor() != null) // to unzip - .forEach(inputFile -> addUnzip(shell, inputFile.getName(0), inputFile.getPreProcessor())); + .forEach(inputFile -> addUnzipFilename(shell, inputFile.getName(0), inputFile.getPreProcessor())); return shell; } @@ -99,7 +103,7 @@ List parser(CommandExecution commandExecution, Path workingDir, Map list, Command command, int executionIndex) { command.getInputFiles().stream() .filter(InputFile::dependsOnExecutionNumber) - .forEach(file -> addUnzip(list, file.getName(executionIndex), file.getPreProcessor())); + .forEach(file -> addUnzipFilename(list, file.getName(executionIndex), file.getPreProcessor())); } private void preProcess(List shell, Command command) { @@ -110,19 +114,29 @@ private void preProcess(List shell, Command command) { // skip because this file is already unzip in a previous batch continue; } - addUnzip(shell, "$" + PRE_FILE + i, inputFile.getPreProcessor()); + addUnzipVariable(shell, PRE_FILE + i, inputFile.getPreProcessor()); } } - private static void addUnzip(List shell, String filename, FilePreProcessor preProcessor) { + // used in batch mode + private static void addUnzipFilename(List shell, String filename, FilePreProcessor preProcessor) { + addUnzipCmd(shell, WRAP_FILENAME.apply(filename), preProcessor); + } + + // used in array mode + private static void addUnzipVariable(List shell, String variableName, FilePreProcessor preProcessor) { + addUnzipCmd(shell, VAR2ARG.apply(variableName), preProcessor); + } + + private static void addUnzipCmd(List shell, String unzipArg, FilePreProcessor preProcessor) { switch (preProcessor) { case FILE_GUNZIP: // gunzip the file - shell.add(SH_GUNZIP + filename); + shell.add(SH_GUNZIP + unzipArg); break; case ARCHIVE_UNZIP: // extract the archive - shell.add(SH_UNZIP + filename); + shell.add(SH_UNZIP + unzipArg); break; default: throw new AssertionError("Unexpected FilePreProcessor value: " + preProcessor); @@ -171,7 +185,7 @@ private void postProcess(List list, Command command, int executionIndex) fileName = file.getName(executionIndex); if (postProcessor != null) { if (postProcessor == FilePostProcessor.FILE_GZIP) { - list.add(SH_GZIP + fileName); + list.add(SH_GZIP + WRAP_FILENAME.apply(fileName)); } else { throw new AssertionError("Unexpected postProcessor type value: " + postProcessor); } @@ -184,15 +198,15 @@ private void postProcess(List shell, Command command) { for (int i = 0; i < outputFiles.size(); i++) { OutputFile outputFile = outputFiles.get(i); FilePostProcessor postProcessor = outputFile.getPostProcessor(); - addGzip(shell, "$" + POST_FILE + i, postProcessor); + addGzipVariable(shell, POST_FILE + i, postProcessor); // TODO add touch my error for gzip command?? } } - private static void addGzip(List shell, String fileName, FilePostProcessor postProcessor) { + private static void addGzipVariable(List shell, String variableName, FilePostProcessor postProcessor) { if (postProcessor != null) { if (postProcessor == FilePostProcessor.FILE_GZIP) { - shell.add(SH_GZIP + fileName); + shell.add(SH_GZIP + VAR2ARG.apply(variableName)); } else { throw new AssertionError("Unexpected postProcessor type value: " + postProcessor); } @@ -248,7 +262,7 @@ private void addInputFilenames(List shell, int caseIdx, Command command) // skip because this file is already unzip in a previous batch continue; } - ins.add(PRE_FILE + i + "=" + inputFile.getName(caseIdx)); + ins.add(PRE_FILE + i + "=" + WRAP_FILENAME.apply(inputFile.getName(caseIdx))); } if (ins.isEmpty()) { return; @@ -264,7 +278,7 @@ private void addOutputFilenames(List shell, int caseIdx, Command command if (!outputFile.dependsOnExecutionNumber() || outputFile.getPostProcessor() == null) { continue; } - outs.add(POST_FILE + i + "=" + outputFile.getName(caseIdx)); + outs.add(POST_FILE + i + "=" + WRAP_FILENAME.apply(outputFile.getName(caseIdx))); } if (outs.isEmpty()) { return; diff --git a/computation-slurm/src/test/java/com/powsybl/computation/slurm/CommandExecutionsTestFactory.java b/computation-slurm/src/test/java/com/powsybl/computation/slurm/CommandExecutionsTestFactory.java index 7a65006..68bd3a2 100644 --- a/computation-slurm/src/test/java/com/powsybl/computation/slurm/CommandExecutionsTestFactory.java +++ b/computation-slurm/src/test/java/com/powsybl/computation/slurm/CommandExecutionsTestFactory.java @@ -69,6 +69,18 @@ static List myEchoSimpleCmdWithUnzipZip(int executionCount) { return Collections.singletonList(commandExecution); } + static List testFilesWithSpaces(int executionCount) { + Command command = new SimpleCommandBuilder() + .id("rename") + .program("mv") + .inputFiles(new InputFile(integer -> "in " + integer + ".zip", FilePreProcessor.ARCHIVE_UNZIP)) + .outputFiles(new OutputFile(integer -> "out " + integer, FilePostProcessor.FILE_GZIP)) + .args(i -> Arrays.asList("in " + i, "out " + i)) + .build(); + CommandExecution commandExecution = new CommandExecution(command, executionCount); + return Collections.singletonList(commandExecution); + } + static List commandFiles(int executionCount) { InputFile stringInput = new InputFile("foo.zip", FilePreProcessor.ARCHIVE_UNZIP); InputFile functionsInput = new InputFile(integer -> "in" + integer + ".zip", FilePreProcessor.ARCHIVE_UNZIP); diff --git a/computation-slurm/src/test/java/com/powsybl/computation/slurm/SbatchScriptGeneratorTest.java b/computation-slurm/src/test/java/com/powsybl/computation/slurm/SbatchScriptGeneratorTest.java index 49705d4..15db1f1 100644 --- a/computation-slurm/src/test/java/com/powsybl/computation/slurm/SbatchScriptGeneratorTest.java +++ b/computation-slurm/src/test/java/com/powsybl/computation/slurm/SbatchScriptGeneratorTest.java @@ -114,10 +114,10 @@ public void testMyEchoSimpleCmd() { Command command = commandExecution.getCommand(); List shell = new SbatchScriptGenerator(flagPath).parser(command, 0, workingPath, Collections.emptyMap()); assertEquals(ImmutableList.of("#!/bin/sh", - "unzip -o -q in0.zip", + "unzip -o -q 'in0.zip'", "/home/dev-itesla/myapps/myecho.sh \"in0\" \"out0\"", "rc=$?; if [[ $rc != 0 ]]; then touch /tmp/flags/myerror_workingPath_12345_$SLURM_JOBID; exit $rc; fi", - "gzip out0", + "gzip 'out0'", "touch /tmp/flags/mydone_workingPath_12345_$SLURM_JOBID"), shell); assertCommandExecutionToShell(commandExecutions.get(0), "myEchoSimpleCmdWithUnzipZip.batch"); @@ -134,10 +134,10 @@ public void testCommandFiles() { private static List expectedTestCommandFilesBatch() { List shell = new ArrayList<>(); shell.add("#!/bin/sh"); - shell.add("unzip -o -q in2.zip"); + shell.add("unzip -o -q 'in2.zip'"); shell.add("/home/dev-itesla/myapps/myecho.sh \"in2\" \"out2\""); shell.add("rc=$?; if [[ $rc != 0 ]]; then touch /tmp/flags/myerror_workingPath_12345_$SLURM_JOBID; exit $rc; fi"); - shell.add("gzip tozip2"); + shell.add("gzip 'tozip2'"); shell.add("touch /tmp/flags/mydone_workingPath_12345_$SLURM_JOBID"); return shell; } @@ -153,7 +153,7 @@ public void testOnlyUnzipBatch() { private static List expectedtestOnlyUnzipBatch() { List shell = new ArrayList<>(); shell.add("#!/bin/sh"); - shell.add("unzip -o -q foo.zip"); + shell.add("unzip -o -q 'foo.zip'"); return shell; } diff --git a/computation-slurm/src/test/resources/expectedShell/myEchoSimpleCmdWithUnzipZip.batch b/computation-slurm/src/test/resources/expectedShell/myEchoSimpleCmdWithUnzipZip.batch index 15b7b58..890e23a 100644 --- a/computation-slurm/src/test/resources/expectedShell/myEchoSimpleCmdWithUnzipZip.batch +++ b/computation-slurm/src/test/resources/expectedShell/myEchoSimpleCmdWithUnzipZip.batch @@ -1,22 +1,22 @@ #!/bin/sh case $SLURM_ARRAY_TASK_ID in 0) - PRE0=in0.zip - POST0=out0 + PRE0='in0.zip' + POST0='out0' ARGS=("in0" "out0") ;; 1) - PRE0=in1.zip - POST0=out1 + PRE0='in1.zip' + POST0='out1' ARGS=("in1" "out1") ;; 2) - PRE0=in2.zip - POST0=out2 + PRE0='in2.zip' + POST0='out2' ARGS=("in2" "out2") ;; esac -unzip -o -q $PRE0 +unzip -o -q "$PRE0" /home/dev-itesla/myapps/myecho.sh "${ARGS[@]}" rc=$?; if [[ $rc != 0 ]]; then touch /tmp/flags/myerror_workingPath_12345_$SLURM_ARRAY_JOB_ID-$SLURM_ARRAY_TASK_ID; exit $rc; fi -gzip $POST0 +gzip "$POST0"