diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java index 42e79ea95..6b5b0bb9f 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java @@ -48,21 +48,6 @@ public void setMaxRamOverride(Integer maxRamOverride) _maxRamOverride = maxRamOverride; } - public void addJava8HomeToEnvironment() - { - //since GATK requires java8, set JAVA_HOME to match this: - File java8 = new File(SequencePipelineService.get().getJava8FilePath()).getParentFile(); - if (java8.getParentFile() == null) - { - getLogger().debug("unexpected path to java8, cannot determine JAVA_HOME: " + java8.getPath()); - return; - } - - String javaDir = java8.getParentFile().getPath(); - getLogger().debug("setting JAVA_HOME to java8 location: " + javaDir); - addToEnvironment("JAVA_HOME", javaDir); - } - public boolean jarExists() { return getJAR(false) != null; @@ -80,7 +65,7 @@ protected void ensureDictionary(File referenceFasta) throws PipelineJobException public String getVersionString() throws PipelineJobException { List args = new ArrayList<>(); - args.add(SequencePipelineService.get().getJava8FilePath()); + args.add(SequencePipelineService.get().getJavaFilepath()); args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride)); args.add("-jar"); args.add(getJAR().getPath()); @@ -102,7 +87,7 @@ protected String getPackageName() public List getBaseArgs(@Nullable String toolName) { List args = new ArrayList<>(); - args.add(SequencePipelineService.get().getJava8FilePath()); + args.add(SequencePipelineService.get().getJavaFilepath()); args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride)); args.add("-jar"); args.add(getJAR().getPath()); diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatkWrapper.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatkWrapper.java index e19ca9fd6..98eab53bf 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatkWrapper.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatkWrapper.java @@ -54,38 +54,6 @@ public void setMaxRamOverride(Integer maxRamOverride) _maxRamOverride = maxRamOverride; } - protected void addJavaHomeToEnvironment() - { - //since GATK requires java8, set JAVA_HOME to match this: - File java8 = new File(SequencePipelineService.get().getJava8FilePath()).getParentFile(); - if (java8.getParentFile() == null) - { - getLogger().debug("unexpected path to java8, cannot determine JAVA_HOME: " + java8.getPath()); - return; - } - - String javaDir = java8.getParentFile().getPath(); - getLogger().debug("setting JAVA_HOME to java8 location: " + javaDir); - addToEnvironment("JAVA_HOME", javaDir); - } - - protected File getQueueJAR() - { - String path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("GATKPATH"); - if (path != null) - { - return new File(path); - } - - path = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath(SequencePipelineService.SEQUENCE_TOOLS_PARAM); - if (path == null) - { - path = PipelineJobService.get().getAppProperties().getToolsDirectory(); - } - - return path == null ? new File("Queue.jar") : new File(path, "Queue.jar"); - } - public boolean jarExists() { return getJAR() == null || !getJAR().exists(); @@ -100,69 +68,6 @@ protected void ensureDictionary(File referenceFasta) throws PipelineJobException new CreateSequenceDictionaryWrapper(getLogger()).execute(referenceFasta, false); } - public String getVersionString() throws PipelineJobException - { - List args = new ArrayList<>(); - args.add(SequencePipelineService.get().getJava8FilePath()); - args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride)); - args.add("-jar"); - args.add(getJAR().getPath()); - args.add("--version"); - - return StringUtils.trimToNull(executeWithOutput(args)); - } - - public Integer getMinRamPerQueueJob() - { - return _minRamPerQueueJob; - } - - public void setMinRamPerQueueJob(Integer minRamPerQueueJob) - { - _minRamPerQueueJob = minRamPerQueueJob; - } - - protected Integer getScatterForQueueJob() - { - // NOTE: Queue will create n number of jobs, dividing memory evenly between them. Because it is possible - // to submit a job w/ lower available RAM and comparably high CPUs, this could result in queue not having enough memory per job. - // therefore do a quick check and potentially scale down scatter - Integer maxThreads = SequencePipelineService.get().getMaxThreads(getLogger()); - if (maxThreads != null) - { - if (_minRamPerQueueJob != null && _minRamPerQueueJob > 0) - { - String maxRamSetting = StringUtils.trimToNull(System.getenv("SEQUENCEANALYSIS_MAX_RAM")); - if (maxRamSetting != null) - { - try - { - Integer maxRamAllowed = ConvertHelper.convert(maxRamSetting, Integer.class); - if (maxRamAllowed != null) - { - int adjusted = Math.max(maxRamAllowed / _minRamPerQueueJob, 1); - if (adjusted < maxThreads) - { - getLogger().debug("lowering max threads to match available RAM. setting to: " + adjusted); - maxThreads = adjusted; - } - } - } - catch (ConvergenceException e) - { - getLogger().warn("non-numeric value for SEQUENCEANALYSIS_MAX_RAM: [" + maxRamSetting + "]"); - } - } - } - } - else - { - maxThreads = 1; - } - - return maxThreads; - } - protected List getBaseArgs() { List args = new ArrayList<>(); diff --git a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/PicardWrapper.java b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/PicardWrapper.java index c48c383f7..63404c554 100644 --- a/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/PicardWrapper.java +++ b/SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/PicardWrapper.java @@ -36,7 +36,7 @@ public String getVersion() throws PipelineJobException } List params = new LinkedList<>(); - params.add(SequencePipelineService.get().getJava8FilePath()); + params.add(SequencePipelineService.get().getJavaFilepath()); params.add("-jar"); params.add(getJar().getPath()); params.add(getToolName()); @@ -95,7 +95,7 @@ protected List getBaseArgs(boolean basicArgsOnly) throws PipelineJobExce } List params = new LinkedList<>(); - params.add(SequencePipelineService.get().getJava8FilePath()); + params.add(SequencePipelineService.get().getJavaFilepath()); params.addAll(SequencePipelineService.get().getJavaOpts()); params.add("-jar"); params.add(getJar().getPath()); diff --git a/SequenceAnalysis/pipeline_code/sequence_tools_install.sh b/SequenceAnalysis/pipeline_code/sequence_tools_install.sh index cf936da0d..33e41d041 100755 --- a/SequenceAnalysis/pipeline_code/sequence_tools_install.sh +++ b/SequenceAnalysis/pipeline_code/sequence_tools_install.sh @@ -276,10 +276,10 @@ then rm -Rf gatk-4* rm -Rf $LKTOOLS_DIR/GenomeAnalysisTK4.jar - wget $WGET_OPTS https://github.com/broadinstitute/gatk/releases/download/4.3.0.0/gatk-4.3.0.0.zip - unzip gatk-4.3.0.0.zip + wget $WGET_OPTS https://github.com/broadinstitute/gatk/releases/download/4.4.0.0/gatk-4.4.0.0.zip + unzip gatk-4.4.0.0.zip - cp ./gatk-4.3.0.0/gatk-package-4.3.0.0-local.jar $LKTOOLS_DIR/GenomeAnalysisTK4.jar + cp ./gatk-4.4.0.0/gatk-package-4.4.0.0-local.jar $LKTOOLS_DIR/GenomeAnalysisTK4.jar else echo "Already installed" fi @@ -695,7 +695,7 @@ then rm -Rf $LKTOOLS_DIR/htsjdk-* rm -Rf $LKTOOLS_DIR/libIntelDeflater.so - wget $WGET_OPTS https://github.com/broadinstitute/picard/releases/download/2.27.4/picard.jar + wget $WGET_OPTS https://github.com/broadinstitute/picard/releases/download/3.0.0/picard.jar cp -R ./picard.jar $LKTOOLS_DIR/ else diff --git a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/bampostprocessing/IndelRealignerStep.java b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/bampostprocessing/IndelRealignerStep.java index f7b8208bd..887ecead8 100644 --- a/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/bampostprocessing/IndelRealignerStep.java +++ b/SequenceAnalysis/src/org/labkey/sequenceanalysis/run/bampostprocessing/IndelRealignerStep.java @@ -16,6 +16,7 @@ import java.io.File; import java.util.Arrays; +import java.util.Collections; /** * User: bimber @@ -24,7 +25,7 @@ */ public class IndelRealignerStep extends AbstractCommandPipelineStep implements BamProcessingStep { - public IndelRealignerStep(PipelineStepProvider provider, PipelineContext ctx) + public IndelRealignerStep(PipelineStepProvider provider, PipelineContext ctx) { super(provider, ctx, new IndelRealignerWrapper(ctx.getLogger())); } @@ -33,17 +34,7 @@ public static class Provider extends AbstractPipelineStepProvider tempFiles = new ArrayList<>(); - File workingBam = performSharedWork(inputBam, outputBam, referenceFasta, tempFiles); - if (!workingBam.equals(inputBam)) - { - tempFiles.add(workingBam); - } - - File intervalsFile = buildTargetIntervals(referenceFasta, workingBam, knownIndelsVcf, getExpectedIntervalsFile(inputBam)); - if (intervalsFile == null) - { - getLogger().info("no intervals to realign, skipping"); - return processOutput(tempFiles, inputBam, outputBam, null); - } - - try - { - Module module = ModuleLoader.getInstance().getModule(SequenceAnalysisModule.class); - FileResource r = (FileResource)module.getModuleResolver().lookup(Path.parse("external/qscript/IndelRealignerRunner.scala")); - File scalaScript = r.getFile(); - - if (scalaScript == null) - throw new FileNotFoundException("Not found: " + scalaScript); - - if (!scalaScript.exists()) - throw new FileNotFoundException("Not found: " + scalaScript.getPath()); - - List args = new ArrayList<>(); - args.add(SequencePipelineService.get().getJava8FilePath()); - //for now, ignore java opts since queue's scatter/gather causes issues - //args.addAll(SequencePipelineService.get().getJavaOpts()); - args.add("-classpath"); - args.add(getJAR().getPath()); - args.addAll(SequencePipelineService.get().getJavaOpts()); - args.add("-jar"); - args.add(getQueueJAR().getPath()); - args.add("-S"); - args.add(scalaScript.getPath()); - args.add("-jobRunner"); - args.add("ParallelShell"); - args.add("-run"); - - args.add("-R"); - args.add(referenceFasta.getPath()); - args.add("-I"); - args.add(workingBam.getPath()); - args.add("-targetIntervals"); - args.add(intervalsFile.getPath()); - - args.add("-runDir"); - args.add(outputBam.getParentFile().getPath()); - - String tmpDir = PipelineJobService.get().getConfigProperties().getSoftwarePackagePath("JAVA_TMP_DIR"); - if (StringUtils.trimToNull(tmpDir) != null) - { - args.add("-tempDir"); - args.add(tmpDir); - } - - args.add("-o"); - - File realignedBam = outputBam == null ? new File(getOutputDir(inputBam), FileUtil.getBaseName(inputBam) + ".realigned.bam") : outputBam; - args.add(realignedBam.getPath()); - - args.add("-startFromScratch"); - args.add("-scatterCount"); - args.add(getScatterForQueueJob().toString()); - - execute(args); - if (!realignedBam.exists()) - { - throw new PipelineJobException("Expected output not found: " + realignedBam.getPath()); - } - - return processOutput(tempFiles, inputBam, outputBam, realignedBam); - } - catch (IOException e) - { - throw new PipelineJobException(e); - } - } - private File processOutput(List tempFiles, File inputBam, File outputBam, File realignedBam) throws PipelineJobException { if (!tempFiles.isEmpty()) diff --git a/jbrowse/src/org/labkey/jbrowse/model/JsonFile.java b/jbrowse/src/org/labkey/jbrowse/model/JsonFile.java index e23f2a762..9a9f0e161 100644 --- a/jbrowse/src/org/labkey/jbrowse/model/JsonFile.java +++ b/jbrowse/src/org/labkey/jbrowse/model/JsonFile.java @@ -972,8 +972,6 @@ private void prepareLuceneIndex(Logger log) throws PipelineJobException log.debug("Generating VCF full text index for file: " + getExpData().getFile().getName()); DISCVRSeqRunner runner = new DISCVRSeqRunner(log); - runner.addJava8HomeToEnvironment(); - if (!runner.jarExists()) { log.error("Unable to find DISCVRSeq.jar, skiping lucene index creation"); @@ -994,10 +992,6 @@ private void prepareLuceneIndex(Logger log) throws PipelineJobException args.add(field); } - // Always include this: - args.add("-AN"); - args.add("SampleList"); - runner.execute(args); }