Skip to content

Commit

Permalink
Merge pull request #318 from LabKey/fb_filesystemlike
Browse files Browse the repository at this point in the history 
FileSystemLike
  • Loading branch information
@labkey-matthewb
labkey-matthewb authored Oct 10, 2024
2 parents 89c1616 + 9c9106f commit 20a9f93
Show file tree
Hide file tree
Showing 6 changed files with 75 additions and 60 deletions.
41 changes: 21 additions & 20 deletions SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@
import org.labkey.sequenceanalysis.util.ChainFileValidator;
import org.labkey.sequenceanalysis.util.FastqUtils;
import org.labkey.sequenceanalysis.util.SequenceUtil;
import org.labkey.vfs.FileLike;
import org.springframework.beans.PropertyValues;
import org.springframework.validation.BindException;
import org.springframework.validation.Errors;
Expand Down Expand Up @@ -2016,30 +2017,30 @@ else if (!d.getFile().exists())
}
else if (o.has("relPath") || o.has("fileName"))
{
File f;
FileLike f;
if (o.opt("relPath") == null)
{
if (path != null)
{
f = pr.resolvePath(path);
f = new File(f, o.getString("fileName"));
f = pr.resolvePathToFileLike(path);
f = f.resolveFile(Path.parse(o.getString("fileName")));
}
else
{
f = pr.resolvePath(o.getString("fileName"));
f = pr.resolvePathToFileLike(o.getString("fileName"));
}
}
else
{
f = pr.resolvePath(o.getString("relPath"));
f = pr.resolvePathToFileLike(o.getString("relPath"));
}

if (f == null || !f.exists())
{
throw new PipelineValidationException("Unknown file: " + o.getString("relPath") + " / " + o.getString("fileName"));
}

ret.add(f);
ret.add(f.toNioPathForRead().toFile());
}
else if (o.opt("filePath") != null)
{
Expand Down Expand Up @@ -2555,8 +2556,8 @@ protected File getTargetFile(String filename) throws IOException

try
{
File targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer(), "sequenceOutputs");
return AssayFileWriter.findUniqueFileName(filename, targetDirectory);
FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer(), "sequenceOutputs");
return AssayFileWriter.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile();
}
catch (ExperimentException e)
{
Expand Down Expand Up @@ -2688,8 +2689,8 @@ protected File getTargetFile(String filename) throws IOException

try
{
File targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer());
return AssayFileWriter.findUniqueFileName(filename, targetDirectory);
FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer());
return AssayFileWriter.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile();
}
catch (ExperimentException e)
{
Expand Down Expand Up @@ -2853,8 +2854,8 @@ protected File getTargetFile(String filename) throws IOException

try
{
File targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer());
return AssayFileWriter.findUniqueFileName(filename, targetDirectory);
FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer());
return AssayFileWriter.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile();
}
catch (ExperimentException e)
{
Expand Down Expand Up @@ -3643,7 +3644,7 @@ else if (fastaFileType.isType(data.getFile()))
}
}

File dict = new File(data.getFile().getParentFile(), FileUtil.getBaseName(data.getFile()) + ".dict");
File dict = FileUtil.appendName(data.getFile().getParentFile(), FileUtil.getBaseName(data.getFile()) + ".dict");
if (dict.exists())
{
files.add(dict);
Expand Down Expand Up @@ -4142,10 +4143,10 @@ public ApiResponse execute(ImportOutputFilesForm form, BindException errors) thr
return null;
}

File targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer(), "sequenceOutputs");
FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer(), "sequenceOutputs");
if (!targetDirectory.exists())
{
targetDirectory.mkdirs();
targetDirectory.mkdir();
}

if (form.getRecords() != null)
Expand All @@ -4155,7 +4156,7 @@ public ApiResponse execute(ImportOutputFilesForm form, BindException errors) thr
Map<File, Map<String, Object>> toCreate = new HashMap<>();
for (JSONObject o : JsonUtil.toJSONObjectList(arr))
{
File file = new File(dirData, o.getString("fileName"));
File file = FileUtil.appendName(dirData, o.getString("fileName"));
if (!file.exists())
{
errors.reject(ERROR_MSG, "Unknown file: " + file.getPath());
Expand Down Expand Up @@ -4196,8 +4197,8 @@ public ApiResponse execute(ImportOutputFilesForm form, BindException errors) thr

for (File file : toCreate.keySet())
{
File target = AssayFileWriter.findUniqueFileName(file.getName(), targetDirectory);
FileUtils.moveFile(file, target);
FileLike target = AssayFileWriter.findUniqueFileName(file.getName(), targetDirectory);
FileUtils.moveFile(file, target.toNioPathForWrite().toFile());

ExpData data = ExperimentService.get().createData(getContainer(), new DataType("Sequence Output"));
data.setName(file.getName());
Expand Down Expand Up @@ -4236,14 +4237,14 @@ public ApiResponse execute(ImportOutputFilesForm form, BindException errors) thr
}

_log.info("moving associated file: " + idx.getPath() + ", to: " + idxTargetName);
File idxTarget = new File(targetDirectory, idxTargetName);
FileLike idxTarget = targetDirectory.resolveChild(idxTargetName);
if (idxTarget.exists())
{
_log.error("target already exists, skipping: " + idxTargetName);
}
else
{
FileUtils.moveFile(idx, idxTarget);
FileUtils.moveFile(idx, idxTarget.toNioPathForWrite().toFile());

ExpData idxData = ExperimentService.get().createData(getContainer(), new DataType("Sequence Output"));
idxData.setName(idxTarget.getName());
Expand Down
20 changes: 10 additions & 10 deletions SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/PicardAlignmentMetricsHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ public void complete(JobContext ctx, List<SequenceOutputFile> inputs, List<Seque
File outputDir = ((SequenceOutputHandlerJob)ctx.getJob()).getWebserverDir(false);
List<File> metricsFiles = new ArrayList<>();

File mf = new File(outputDir, FileUtil.getBaseName(o.getFile()) + ".summary.metrics");
File mf = FileUtil.appendName(outputDir, FileUtil.getBaseName(o.getFile()) + ".summary.metrics");
if (mf.exists())
{
metricsFiles.add(mf);
Expand All @@ -181,7 +181,7 @@ else if (collectSummary)
throw new PipelineJobException("Missing file: " + mf.getPath());
}

File mf2 = new File(outputDir, FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics");
File mf2 = FileUtil.appendName(outputDir, FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics");
if (mf2.exists())
{
metricsFiles.add(mf2);
Expand All @@ -199,7 +199,7 @@ else if (collectInsertSize)
}
}

File mf3 = new File(outputDir, FileUtil.getBaseName(o.getFile()) + ".wgs.metrics");
File mf3 = FileUtil.appendName(outputDir, FileUtil.getBaseName(o.getFile()) + ".wgs.metrics");
if (mf3.exists())
{
metricsFiles.add(mf3);
Expand All @@ -209,7 +209,7 @@ else if (collectWgs)
throw new PipelineJobException("Missing file: " + mf3.getPath());
}

File mf4 = new File(outputDir, FileUtil.getBaseName(o.getFile()) + ".wgsNonZero.metrics");
File mf4 = FileUtil.appendName(outputDir, FileUtil.getBaseName(o.getFile()) + ".wgsNonZero.metrics");
if (mf4.exists())
{
metricsFiles.add(mf4);
Expand Down Expand Up @@ -284,7 +284,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
{
job.getLogger().info("calculating summary metrics");
job.setStatus(PipelineJob.TaskStatus.running, "CALCULATING SUMMARY METRICS");
File metricsFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".summary.metrics");
File metricsFile = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".summary.metrics");
AlignmentSummaryMetricsWrapper wrapper = new AlignmentSummaryMetricsWrapper(job.getLogger());
wrapper.executeCommand(o.getFile(), ctx.getSequenceSupport().getCachedGenome(o.getLibrary_id()).getWorkingFastaFile(), metricsFile);
}
Expand All @@ -293,7 +293,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
{
job.getLogger().info("calculating wgs metrics");
job.setStatus(PipelineJob.TaskStatus.running, "CALCULATING WGS METRICS");
File wgsMetricsFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".wgs.metrics");
File wgsMetricsFile = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".wgs.metrics");
CollectWgsMetricsWrapper wgsWrapper = new CollectWgsMetricsWrapper(job.getLogger());
wgsWrapper.executeCommand(o.getFile(), wgsMetricsFile, ctx.getSequenceSupport().getCachedGenome(o.getLibrary_id()).getWorkingFastaFile());
}
Expand All @@ -302,7 +302,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
{
job.getLogger().info("calculating wgs metrics over non zero positions");
job.setStatus(PipelineJob.TaskStatus.running, "CALCULATING WGS METRICS");
File wgsMetricsFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".wgsNonZero.metrics");
File wgsMetricsFile = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".wgsNonZero.metrics");
CollectWgsMetricsWithNonZeroCoverageWrapper wgsWrapper = new CollectWgsMetricsWithNonZeroCoverageWrapper(job.getLogger());
wgsWrapper.executeCommand(o.getFile(), wgsMetricsFile, ctx.getSequenceSupport().getCachedGenome(o.getLibrary_id()).getWorkingFastaFile());
}
Expand All @@ -311,8 +311,8 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
{
job.getLogger().info("calculating insert size metrics");
job.setStatus(PipelineJob.TaskStatus.running, "CALCULATING INSERT SIZE METRICS");
File metricsFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics");
File metricsHistogram = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics.pdf");
File metricsFile = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics");
File metricsHistogram = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics.pdf");
CollectInsertSizeMetricsWrapper wrapper = new CollectInsertSizeMetricsWrapper(job.getLogger());
wrapper.executeCommand(o.getFile(), metricsFile, metricsHistogram, ctx.getSequenceSupport().getCachedGenome(o.getLibrary_id()).getWorkingFastaFile());
}
Expand All @@ -323,7 +323,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
job.setStatus(PipelineJob.TaskStatus.running, "RUNNING MARKDUPLICATES");
MarkDuplicatesWrapper wrapper = new MarkDuplicatesWrapper(job.getLogger());
File metricsFile = wrapper.getMetricsFile(o.getFile());
File tempBam = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".markDuplicates.bam");
File tempBam = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".markDuplicates.bam");
ctx.getFileManager().addIntermediateFile(tempBam);
ctx.getFileManager().addIntermediateFile(SequenceUtil.getExpectedIndex(tempBam));

Expand Down
8 changes: 5 additions & 3 deletions SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceLibraryPipelineJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import org.labkey.sequenceanalysis.SequenceAnalysisManager;
import org.labkey.sequenceanalysis.SequenceAnalysisSchema;
import org.labkey.sequenceanalysis.model.ReferenceLibraryMember;
import org.labkey.vfs.FileLike;
import org.labkey.vfs.FileSystemLike;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -107,7 +109,7 @@ protected void writeParameters(JSONObject params) throws IOException

public File getSerializedLibraryMembersFile()
{
return new File(getDataDirectory(), FileUtil.getBaseName(getLogFile()) + ".json");
return FileUtil.appendName(getDataDirectory(), FileUtil.getBaseName(getLogFile()) + ".json");
}

//for recreating an existing library
Expand Down Expand Up @@ -193,7 +195,7 @@ public boolean isCreateNew()
}

@Override
protected File createLocalDirectory(PipeRoot pipeRoot)
protected FileLike createLocalDirectory(PipeRoot pipeRoot)
{
if (PipelineJobService.get().getLocationType() != PipelineJobService.LocationType.WebServer)
{
Expand All @@ -211,7 +213,7 @@ protected File createLocalDirectory(PipeRoot pipeRoot)
outputDir.mkdirs();
}

return outputDir;
return new FileSystemLike.Builder(outputDir).tempDir().root();
}

@Override
Expand Down
Loading

0 comments on commit 20a9f93

Please sign in to comment.