Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FileSystemLike #318

Merged
merged 6 commits into from
Oct 10, 2024
Merged

FileSystemLike #318

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
54a78c8
AssayDataHandler and AssayUpload conversion File->FileObject
labkey-matthewb Sep 18, 2024
d4bb26a
Merge remote-tracking branch 'origin/develop' into fb_filesystemlike
labkey-matthewb Sep 26, 2024
1aab1df
FileSystemLike and FileLike interfaces
labkey-matthewb Sep 26, 2024
d84a997
Merge remote-tracking branch 'origin/develop' into fb_filesystemlike
labkey-matthewb Oct 1, 2024
e6ecde0
safely handle relative path
labkey-matthewb Oct 4, 2024
9c9106f
Merge remote-tracking branch 'origin/develop' into fb_filesystemlike
labkey-matthewb Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 21 additions & 20 deletions SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@
import org.labkey.sequenceanalysis.util.ChainFileValidator;
import org.labkey.sequenceanalysis.util.FastqUtils;
import org.labkey.sequenceanalysis.util.SequenceUtil;
import org.labkey.vfs.FileLike;
import org.springframework.beans.PropertyValues;
import org.springframework.validation.BindException;
import org.springframework.validation.Errors;
Expand Down Expand Up @@ -2016,30 +2017,30 @@ else if (!d.getFile().exists())
}
else if (o.has("relPath") || o.has("fileName"))
{
File f;
FileLike f;
if (o.opt("relPath") == null)
{
if (path != null)
{
f = pr.resolvePath(path);
f = new File(f, o.getString("fileName"));
f = pr.resolvePathToFileLike(path);
f = f.resolveFile(Path.parse(o.getString("fileName")));
}
else
{
f = pr.resolvePath(o.getString("fileName"));
f = pr.resolvePathToFileLike(o.getString("fileName"));
}
}
else
{
f = pr.resolvePath(o.getString("relPath"));
f = pr.resolvePathToFileLike(o.getString("relPath"));
}

if (f == null || !f.exists())
{
throw new PipelineValidationException("Unknown file: " + o.getString("relPath") + " / " + o.getString("fileName"));
}

ret.add(f);
ret.add(f.toNioPathForRead().toFile());
}
else if (o.opt("filePath") != null)
{
Expand Down Expand Up @@ -2555,8 +2556,8 @@ protected File getTargetFile(String filename) throws IOException

try
{
File targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer(), "sequenceOutputs");
return AssayFileWriter.findUniqueFileName(filename, targetDirectory);
FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer(), "sequenceOutputs");
return AssayFileWriter.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile();
}
catch (ExperimentException e)
{
Expand Down Expand Up @@ -2688,8 +2689,8 @@ protected File getTargetFile(String filename) throws IOException

try
{
File targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer());
return AssayFileWriter.findUniqueFileName(filename, targetDirectory);
FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer());
return AssayFileWriter.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile();
}
catch (ExperimentException e)
{
Expand Down Expand Up @@ -2853,8 +2854,8 @@ protected File getTargetFile(String filename) throws IOException

try
{
File targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer());
return AssayFileWriter.findUniqueFileName(filename, targetDirectory);
FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer());
return AssayFileWriter.findUniqueFileName(filename, targetDirectory).toNioPathForWrite().toFile();
}
catch (ExperimentException e)
{
Expand Down Expand Up @@ -3643,7 +3644,7 @@ else if (fastaFileType.isType(data.getFile()))
}
}

File dict = new File(data.getFile().getParentFile(), FileUtil.getBaseName(data.getFile()) + ".dict");
File dict = FileUtil.appendName(data.getFile().getParentFile(), FileUtil.getBaseName(data.getFile()) + ".dict");
if (dict.exists())
{
files.add(dict);
Expand Down Expand Up @@ -4142,10 +4143,10 @@ public ApiResponse execute(ImportOutputFilesForm form, BindException errors) thr
return null;
}

File targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer(), "sequenceOutputs");
FileLike targetDirectory = AssayFileWriter.ensureUploadDirectory(getContainer(), "sequenceOutputs");
if (!targetDirectory.exists())
{
targetDirectory.mkdirs();
targetDirectory.mkdir();
}

if (form.getRecords() != null)
Expand All @@ -4155,7 +4156,7 @@ public ApiResponse execute(ImportOutputFilesForm form, BindException errors) thr
Map<File, Map<String, Object>> toCreate = new HashMap<>();
for (JSONObject o : JsonUtil.toJSONObjectList(arr))
{
File file = new File(dirData, o.getString("fileName"));
File file = FileUtil.appendName(dirData, o.getString("fileName"));
if (!file.exists())
{
errors.reject(ERROR_MSG, "Unknown file: " + file.getPath());
Expand Down Expand Up @@ -4196,8 +4197,8 @@ public ApiResponse execute(ImportOutputFilesForm form, BindException errors) thr

for (File file : toCreate.keySet())
{
File target = AssayFileWriter.findUniqueFileName(file.getName(), targetDirectory);
FileUtils.moveFile(file, target);
FileLike target = AssayFileWriter.findUniqueFileName(file.getName(), targetDirectory);
FileUtils.moveFile(file, target.toNioPathForWrite().toFile());

ExpData data = ExperimentService.get().createData(getContainer(), new DataType("Sequence Output"));
data.setName(file.getName());
Expand Down Expand Up @@ -4236,14 +4237,14 @@ public ApiResponse execute(ImportOutputFilesForm form, BindException errors) thr
}

_log.info("moving associated file: " + idx.getPath() + ", to: " + idxTargetName);
File idxTarget = new File(targetDirectory, idxTargetName);
FileLike idxTarget = targetDirectory.resolveChild(idxTargetName);
if (idxTarget.exists())
{
_log.error("target already exists, skipping: " + idxTargetName);
}
else
{
FileUtils.moveFile(idx, idxTarget);
FileUtils.moveFile(idx, idxTarget.toNioPathForWrite().toFile());

ExpData idxData = ExperimentService.get().createData(getContainer(), new DataType("Sequence Output"));
idxData.setName(idxTarget.getName());
Expand Down
20 changes: 10 additions & 10 deletions SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/PicardAlignmentMetricsHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ public void complete(JobContext ctx, List<SequenceOutputFile> inputs, List<Seque
File outputDir = ((SequenceOutputHandlerJob)ctx.getJob()).getWebserverDir(false);
List<File> metricsFiles = new ArrayList<>();

File mf = new File(outputDir, FileUtil.getBaseName(o.getFile()) + ".summary.metrics");
File mf = FileUtil.appendName(outputDir, FileUtil.getBaseName(o.getFile()) + ".summary.metrics");
if (mf.exists())
{
metricsFiles.add(mf);
Expand All @@ -181,7 +181,7 @@ else if (collectSummary)
throw new PipelineJobException("Missing file: " + mf.getPath());
}

File mf2 = new File(outputDir, FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics");
File mf2 = FileUtil.appendName(outputDir, FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics");
if (mf2.exists())
{
metricsFiles.add(mf2);
Expand All @@ -199,7 +199,7 @@ else if (collectInsertSize)
}
}

File mf3 = new File(outputDir, FileUtil.getBaseName(o.getFile()) + ".wgs.metrics");
File mf3 = FileUtil.appendName(outputDir, FileUtil.getBaseName(o.getFile()) + ".wgs.metrics");
if (mf3.exists())
{
metricsFiles.add(mf3);
Expand All @@ -209,7 +209,7 @@ else if (collectWgs)
throw new PipelineJobException("Missing file: " + mf3.getPath());
}

File mf4 = new File(outputDir, FileUtil.getBaseName(o.getFile()) + ".wgsNonZero.metrics");
File mf4 = FileUtil.appendName(outputDir, FileUtil.getBaseName(o.getFile()) + ".wgsNonZero.metrics");
if (mf4.exists())
{
metricsFiles.add(mf4);
Expand Down Expand Up @@ -284,7 +284,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
{
job.getLogger().info("calculating summary metrics");
job.setStatus(PipelineJob.TaskStatus.running, "CALCULATING SUMMARY METRICS");
File metricsFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".summary.metrics");
File metricsFile = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".summary.metrics");
AlignmentSummaryMetricsWrapper wrapper = new AlignmentSummaryMetricsWrapper(job.getLogger());
wrapper.executeCommand(o.getFile(), ctx.getSequenceSupport().getCachedGenome(o.getLibrary_id()).getWorkingFastaFile(), metricsFile);
}
Expand All @@ -293,7 +293,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
{
job.getLogger().info("calculating wgs metrics");
job.setStatus(PipelineJob.TaskStatus.running, "CALCULATING WGS METRICS");
File wgsMetricsFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".wgs.metrics");
File wgsMetricsFile = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".wgs.metrics");
CollectWgsMetricsWrapper wgsWrapper = new CollectWgsMetricsWrapper(job.getLogger());
wgsWrapper.executeCommand(o.getFile(), wgsMetricsFile, ctx.getSequenceSupport().getCachedGenome(o.getLibrary_id()).getWorkingFastaFile());
}
Expand All @@ -302,7 +302,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
{
job.getLogger().info("calculating wgs metrics over non zero positions");
job.setStatus(PipelineJob.TaskStatus.running, "CALCULATING WGS METRICS");
File wgsMetricsFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".wgsNonZero.metrics");
File wgsMetricsFile = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".wgsNonZero.metrics");
CollectWgsMetricsWithNonZeroCoverageWrapper wgsWrapper = new CollectWgsMetricsWithNonZeroCoverageWrapper(job.getLogger());
wgsWrapper.executeCommand(o.getFile(), wgsMetricsFile, ctx.getSequenceSupport().getCachedGenome(o.getLibrary_id()).getWorkingFastaFile());
}
Expand All @@ -311,8 +311,8 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
{
job.getLogger().info("calculating insert size metrics");
job.setStatus(PipelineJob.TaskStatus.running, "CALCULATING INSERT SIZE METRICS");
File metricsFile = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics");
File metricsHistogram = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics.pdf");
File metricsFile = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics");
File metricsHistogram = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".insertsize.metrics.pdf");
CollectInsertSizeMetricsWrapper wrapper = new CollectInsertSizeMetricsWrapper(job.getLogger());
wrapper.executeCommand(o.getFile(), metricsFile, metricsHistogram, ctx.getSequenceSupport().getCachedGenome(o.getLibrary_id()).getWorkingFastaFile());
}
Expand All @@ -323,7 +323,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
job.setStatus(PipelineJob.TaskStatus.running, "RUNNING MARKDUPLICATES");
MarkDuplicatesWrapper wrapper = new MarkDuplicatesWrapper(job.getLogger());
File metricsFile = wrapper.getMetricsFile(o.getFile());
File tempBam = new File(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".markDuplicates.bam");
File tempBam = FileUtil.appendName(ctx.getOutputDir(), FileUtil.getBaseName(o.getFile()) + ".markDuplicates.bam");
ctx.getFileManager().addIntermediateFile(tempBam);
ctx.getFileManager().addIntermediateFile(SequenceUtil.getExpectedIndex(tempBam));

Expand Down
8 changes: 5 additions & 3 deletions SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ReferenceLibraryPipelineJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import org.labkey.sequenceanalysis.SequenceAnalysisManager;
import org.labkey.sequenceanalysis.SequenceAnalysisSchema;
import org.labkey.sequenceanalysis.model.ReferenceLibraryMember;
import org.labkey.vfs.FileLike;
import org.labkey.vfs.FileSystemLike;

import java.io.File;
import java.io.IOException;
Expand Down Expand Up @@ -107,7 +109,7 @@ protected void writeParameters(JSONObject params) throws IOException

public File getSerializedLibraryMembersFile()
{
return new File(getDataDirectory(), FileUtil.getBaseName(getLogFile()) + ".json");
return FileUtil.appendName(getDataDirectory(), FileUtil.getBaseName(getLogFile()) + ".json");
}

//for recreating an existing library
Expand Down Expand Up @@ -193,7 +195,7 @@ public boolean isCreateNew()
}

@Override
protected File createLocalDirectory(PipeRoot pipeRoot)
protected FileLike createLocalDirectory(PipeRoot pipeRoot)
{
if (PipelineJobService.get().getLocationType() != PipelineJobService.LocationType.WebServer)
{
Expand All @@ -211,7 +213,7 @@ protected File createLocalDirectory(PipeRoot pipeRoot)
outputDir.mkdirs();
}

return outputDir;
return new FileSystemLike.Builder(outputDir).tempDir().root();
}

@Override
Expand Down
Loading