Merge pull request #988 from aidenlab/hiSummingPreWithStats

SPre with stats

src/juicebox/HiC.java

@@ -26,6 +26,7 @@
 package juicebox;
 
 import juicebox.data.*;
+import juicebox.data.anchor.GenericLocus;
 import juicebox.data.anchor.MotifAnchor;
 import juicebox.data.basics.Chromosome;
 import juicebox.gui.SuperAdapter;
@@ -1238,7 +1239,7 @@ private void clearAllCacheForDataset(Dataset ds) {
         ds.clearCache(false);
     }
 
-    public List<Pair<MotifAnchor, MotifAnchor>> getRTreeHandlerIntersectingFeatures(String name, int g1, int g2) {
+    public List<Pair<GenericLocus, GenericLocus>> getRTreeHandlerIntersectingFeatures(String name, int g1, int g2) {
         try {
             return ((CustomMatrixZoomData) getZd()).getRTreeHandlerIntersectingFeatures(name, g1, g2);
         } catch (Exception ignored) {

src/juicebox/HiCGlobals.java

@@ -1,7 +1,7 @@
 /*
  * The MIT License (MIT)
  *
- * Copyright (c) 2011-2021 Broad Institute, Aiden Lab, Rice University, Baylor College of Medicine
+ * Copyright (c) 2011-2022 Broad Institute, Aiden Lab, Rice University, Baylor College of Medicine
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -36,7 +36,7 @@
  */
 public class HiCGlobals {
 
-    public static final String versionNum = "2.13.07";
+    public static final String versionNum = "2.15.07";
     public static final String juiceboxTitle = "[Juicebox " + versionNum + "] Hi-C Map ";
 
     // MainWindow variables

src/juicebox/data/AbstractDatasetReader.java

@@ -1,7 +1,7 @@
 /*
  * The MIT License (MIT)
  *
- * Copyright (c) 2011-2021 Broad Institute, Aiden Lab, Rice University, Baylor College of Medicine
+ * Copyright (c) 2011-2022 Broad Institute, Aiden Lab, Rice University, Baylor College of Medicine
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -110,10 +110,4 @@ public double[] readEigenvector(String chrName, HiCZoom zoom, int number, String
     public NormalizationVector readNormalizationVector(NormalizationType type, int chrIdx, HiC.Unit unit, int binSize) throws IOException {
         return null;  // Override as necessary
     }
-
-    @Override
-    public String readStats() throws IOException {
-        return null; // Override for Combined Dataset Reader
-    }
-
 }

src/juicebox/data/Block.java

@@ -64,4 +64,8 @@ public String getUniqueRegionID() {
     public List<ContactRecord> getContactRecords() {
         return records;
     }
+
+    public void clear() {
+        records.clear();
+    }
 }

src/juicebox/data/BlockIndex.java

@@ -55,6 +55,14 @@ public List<Integer> getBlockNumbers() {
         return new ArrayList<>(blockIndex.keySet());
     }
 
+    public Integer getBlockSize(int num) {
+        if (blockIndex.containsKey(num)) {
+            return blockIndex.get(num).size;
+        } else {
+            return null;
+        }
+    }
+
     public IndexEntry getBlock(int blockNumber) {
         return blockIndex.get(blockNumber);
     }

src/juicebox/data/ChromosomeHandler.java

@@ -25,9 +25,7 @@
 package juicebox.data;
 
 import juicebox.Context;
-import juicebox.data.anchor.MotifAnchor;
-import juicebox.data.anchor.MotifAnchorParser;
-import juicebox.data.anchor.MotifAnchorTools;
+import juicebox.data.anchor.*;
 import juicebox.data.basics.Chromosome;
 import juicebox.data.feature.FeatureFunction;
 import juicebox.data.feature.GenomeWideList;
@@ -45,7 +43,7 @@ public class ChromosomeHandler {
     public final static int CUSTOM_CHROMOSOME_BUFFER = 10; // todo make to smallest value of resolution
     private static final String CHR_ALL = "All";
     private final Map<String, Chromosome> chromosomeMap = new HashMap<>();
-    private final Map<Integer, GenomeWideList<MotifAnchor>> customChromosomeRegions = new HashMap<>();
+    private final Map<Integer, GenomeWideList<GenericLocus>> customChromosomeRegions = new HashMap<>();
     private final List<Chromosome> cleanedChromosomes;
     private final String genomeID;
     private final long[] chromosomeBoundaries;
@@ -130,13 +128,13 @@ public static boolean isAllByAll(Chromosome chromosome) {
     }
 
 
-    private GenomeWideList<MotifAnchor> generateChromDotSizesBedFile() {
-        GenomeWideList<MotifAnchor> chromDotSizes = new GenomeWideList<>(this);
+    private GenomeWideList<GenericLocus> generateChromDotSizesBedFile() {
+        GenomeWideList<GenericLocus> chromDotSizes = new GenomeWideList<>(this);
 
         for (Chromosome c : getChromosomeArray()) {
             if (isAllByAll(c) || isGenomeWide(c)) continue;
             MotifAnchor chromAnchor = new MotifAnchor(c.getName(), 0, (int) c.getLength(), c.getName()); // not implemented or called
-            List<MotifAnchor> anchors = new ArrayList<>();
+            List<GenericLocus> anchors = new ArrayList<>();
             anchors.add(chromAnchor);
             chromDotSizes.setFeatures("" + c.getIndex(), anchors);
         }
@@ -153,7 +151,7 @@ private boolean isGenomeWide(String name) {
     }
 
     public Chromosome addGenomeWideChromosome() {
-        GenomeWideList<MotifAnchor> chromDotSizes = generateChromDotSizesBedFile();
+        GenomeWideList<GenericLocus> chromDotSizes = generateChromDotSizesBedFile();
         return addCustomChromosome(chromDotSizes, cleanUpName(GENOMEWIDE_CHR));
     }
 
@@ -174,8 +172,8 @@ public Chromosome generateAssemblyChromosome() {
     }
 
     public Chromosome generateCustomChromosomeFromBED(File file, int minSize) {
-        GenomeWideList<MotifAnchor> regionsInCustomChromosome =
-                MotifAnchorParser.loadFromBEDFile(this, file.getAbsolutePath());
+        GenomeWideList<GenericLocus> regionsInCustomChromosome =
+                GenericLocusParser.loadFromBEDFile(this, file.getAbsolutePath());
 
         MotifAnchorTools.mergeAndExpandSmallAnchors(regionsInCustomChromosome, minSize);
 
@@ -185,26 +183,26 @@ public Chromosome generateCustomChromosomeFromBED(File file, int minSize) {
     }
 
     public Chromosome addCustomChromosome(Feature2DList featureList, String chrName) {
-        GenomeWideList<MotifAnchor> featureAnchors =
+        GenomeWideList<GenericLocus> featureAnchors =
                 MotifAnchorTools.extractAllAnchorsFromAllFeatures(featureList, this);
         String cleanedUpName = cleanUpName(chrName);
         return addCustomChromosome(featureAnchors, cleanedUpName);
     }
 
-    private int getTotalLengthOfAllRegionsInBedFile(GenomeWideList<MotifAnchor> regionsInCustomChromosome) {
+    private int getTotalLengthOfAllRegionsInBedFile(GenomeWideList<GenericLocus> regionsInCustomChromosome) {
         final int[] customGenomeLength = new int[]{0};
-        regionsInCustomChromosome.processLists(new FeatureFunction<MotifAnchor>() {
+        regionsInCustomChromosome.processLists(new FeatureFunction<GenericLocus>() {
             @Override
-            public void process(String chr, List<MotifAnchor> featureList) {
-                for (MotifAnchor c : featureList) {
+            public void process(String chr, List<GenericLocus> featureList) {
+                for (GenericLocus c : featureList) {
                     if (c != null) customGenomeLength[0] += c.getWidth() + CUSTOM_CHROMOSOME_BUFFER;
                 }
             }
         });
         return customGenomeLength[0];
     }
 
-    private Chromosome addCustomChromosome(GenomeWideList<MotifAnchor> regionsInCustomChromosome, String cleanedUpName) {
+    private Chromosome addCustomChromosome(GenomeWideList<GenericLocus> regionsInCustomChromosome, String cleanedUpName) {
         int size = getTotalLengthOfAllRegionsInBedFile(regionsInCustomChromosome);
         int newIndex = cleanedChromosomes.size();
         customChromosomeRegions.put(newIndex, regionsInCustomChromosome);
@@ -342,7 +340,7 @@ public Chromosome[] getChromosomeArrayWithoutAllByAll() {
         return chromosomeArrayWithoutAllByAll;
     }
 
-    public GenomeWideList<MotifAnchor> getListOfRegionsInCustomChromosome(Integer index) {
+    public GenomeWideList<GenericLocus> getListOfRegionsInCustomChromosome(Integer index) {
         return customChromosomeRegions.get(index);
     }
 

src/juicebox/data/CombinedDatasetReader.java

@@ -1,7 +1,7 @@
 /*
  * The MIT License (MIT)
  *
- * Copyright (c) 2011-2021 Broad Institute, Aiden Lab, Rice University, Baylor College of Medicine
+ * Copyright (c) 2011-2022 Broad Institute, Aiden Lab, Rice University, Baylor College of Medicine
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -100,12 +100,6 @@ public String getPath() {
         return null;
     }
 
-    @Override
-    public String readStats() {
-        // again we need to somehow combine from constituent datasets
-        return null;
-    }
-
     @Override
     public List<JCheckBox> getCheckBoxes(List<ActionListener> actionListeners) {
         List<JCheckBox> allBoxes = new ArrayList<>();
@@ -157,6 +151,7 @@ public Block readNormalizedBlock(int blockNumber, MatrixZoomData zd, Normalizati
                     blockList.add(cb);
                 }
             }
+
         }
         String key = zd.getBlockKey(blockNumber, no);
         return blockList.size() == 0 ? new Block(blockNumber, key) : mergeBlocks(blockList, key);
@@ -172,7 +167,6 @@ public Block readNormalizedBlock(int blockNumber, MatrixZoomData zd, Normalizati
     @Override
 
     public List<Integer> getBlockNumbers(MatrixZoomData matrixZoomData) {
-
         Set<Integer> blockNumberSet = new HashSet<>();
         for (DatasetReader r : readers) {
             if (r.isActive()) {
@@ -184,6 +178,24 @@ public List<Integer> getBlockNumbers(MatrixZoomData matrixZoomData) {
         return blockNumbers;
     }
 
+    @Override
+    public Integer getBlockSize(MatrixZoomData zd, int blockNum) {
+        Integer blockSize = 0;
+        for (DatasetReader r : readers) {
+            if (r.isActive()) {
+                Integer tmpBlockSize = r.getBlockSize(zd, blockNum);
+                if (tmpBlockSize != null) {
+                    blockSize += tmpBlockSize;
+                }
+            }
+        }
+        if (blockSize != 0) {
+            return blockSize;
+        } else {
+            return null;
+        }
+    }
+
     @Override
     public double[] readEigenvector(String chr, HiCZoom zoom, int number, String type) {
         // Eigenvectors not supported for combined datasets
@@ -496,17 +508,27 @@ private String mergeGraphs(List<String> graphsList) {
     private Matrix mergeMatrices(List<Matrix> matrixList) {
 
         Map<String, Double> averageCount = new HashMap<>();
+        Map<String, Double> sumCount = new HashMap<>();
         List<MatrixZoomData> newMatrixZoomData = new ArrayList<>();
+        Set<Integer> bpBinSizesSet = new HashSet<>();
+        Set<Integer> fragBinSizesSet = new HashSet<>();
         for (Matrix matrix : matrixList) {
             for (MatrixZoomData zd : matrix.bpZoomData) {
                 String key = zd.getKey();
                 Double avg = averageCount.get(key);
+                Double sum = sumCount.get(key);
                 if (avg == null) {
                     averageCount.put(key, zd.getAverageCount());
                 } else if (avg >= 0) {
                     averageCount.put(key, avg + zd.getAverageCount());
                 }
+                if (sum == null) {
+                    sumCount.put(key, zd.getSumCount());
+                } else if (sum >= 0) {
+                    sumCount.put(key, sum + zd.getSumCount());
+                }
                 newMatrixZoomData.add(zd);
+                bpBinSizesSet.add(zd.getBinSize());
             }
             if (hasFrags) {
                 for (MatrixZoomData zd : matrix.fragZoomData) {
@@ -517,18 +539,25 @@ private Matrix mergeMatrices(List<Matrix> matrixList) {
                     } else if (avg >= 0) {
                         averageCount.put(key, avg + zd.getAverageCount());
                     }
+                    fragBinSizesSet.add(zd.getBinSize());
                 }
             }
         }
 
         Matrix mergedMatrix = matrixList.get(0);
+        ArrayList<Integer> bpBinSizes = new ArrayList<>(bpBinSizesSet);
+        Collections.sort(bpBinSizes, Collections.reverseOrder());
+        mergedMatrix.setBpBinSizes(bpBinSizes);
 
         for (MatrixZoomData zd : newMatrixZoomData)       {
             zd.reader = this;
             String key = zd.getKey();
             if (averageCount.containsKey(key)) {
                 zd.setAverageCount(averageCount.get(key));
             }
+            if (sumCount.containsKey(key)) {
+                zd.setSumCount(sumCount.get(key));
+            }
         }
         mergedMatrix.bpZoomData = newMatrixZoomData;
        /* for (MatrixZoomData zd : mergedMatrix.bpZoomData) {
@@ -539,12 +568,18 @@ private Matrix mergeMatrices(List<Matrix> matrixList) {
             }
         }*/
         if (hasFrags) {
+            ArrayList<Integer> fragBinSizes = new ArrayList<>(fragBinSizesSet);
+            Collections.sort(fragBinSizes, Collections.reverseOrder());
+            mergedMatrix.setFragBinSizes(fragBinSizes);
             for (MatrixZoomData zd : mergedMatrix.fragZoomData) {
                 zd.reader = this;
                 String key = zd.getKey();
                 if (averageCount.containsKey(key)) {
                     zd.setAverageCount(averageCount.get(key));
                 }
+                if (sumCount.containsKey(key)) {
+                    zd.setSumCount(sumCount.get(key));
+                }
             }
         } else {
             mergedMatrix.fragZoomData = null;

src/juicebox/data/CustomMatrixZoomData.java

@@ -25,6 +25,7 @@
 package juicebox.data;
 
 import juicebox.HiCGlobals;
+import juicebox.data.anchor.GenericLocus;
 import juicebox.data.anchor.MotifAnchor;
 import juicebox.data.basics.Chromosome;
 import juicebox.data.censoring.CustomMZDRegionHandler;
@@ -129,11 +130,11 @@ private List<Block> addNormalizedBlocksToListByGenomeCoordinates(long gx1, long
 
         // x window
         //net.sf.jsi.Rectangle currentWindow = new net.sf.jsi.Rectangle(gx1, gx1, gx2, gx2);
-        List<Pair<MotifAnchor, MotifAnchor>> xAxisRegions = rTreeHandler.getIntersectingFeatures(chr1.getName(), gx1, gx2);
+        List<Pair<GenericLocus, GenericLocus>> xAxisRegions = rTreeHandler.getIntersectingFeatures(chr1.getName(), gx1, gx2);
 
         // y window
         //currentWindow = new net.sf.jsi.Rectangle(gy1, gy1, gy2, gy2);
-        List<Pair<MotifAnchor, MotifAnchor>> yAxisRegions = rTreeHandler.getIntersectingFeatures(chr2.getName(), gy1, gy2);
+        List<Pair<GenericLocus, GenericLocus>> yAxisRegions = rTreeHandler.getIntersectingFeatures(chr2.getName(), gy1, gy2);
 
         if (isImportant) {
             if (HiCGlobals.printVerboseComments)
@@ -149,8 +150,8 @@ private List<Block> addNormalizedBlocksToListByGenomeCoordinates(long gx1, long
 
         ExecutorService executor = HiCGlobals.newFixedThreadPool();
         // todo change to be by chromosome?
-        for (Pair<MotifAnchor, MotifAnchor> xRegion : xAxisRegions) {
-            for (Pair<MotifAnchor, MotifAnchor> yRegion : yAxisRegions) {
+        for (Pair<GenericLocus, GenericLocus> xRegion : xAxisRegions) {
+            for (Pair<GenericLocus, GenericLocus> yRegion : yAxisRegions) {
                 Runnable worker = new Runnable() {
                     @Override
                     public void run() {
@@ -326,12 +327,12 @@ public double getExpected(int binX, int binY, ExpectedValueFunction df) {
         // x window
         int gx1 = binX * zoom.getBinSize();
         net.sf.jsi.Rectangle currentWindow = new net.sf.jsi.Rectangle(gx1, gx1, gx1, gx1);
-        List<Pair<MotifAnchor, MotifAnchor>> xRegions = rTreeHandler.getIntersectingFeatures(chr1.getName(), gx1);
+        List<Pair<GenericLocus, GenericLocus>> xRegions = rTreeHandler.getIntersectingFeatures(chr1.getName(), gx1);
 
         // y window
         int gy1 = binY * zoom.getBinSize();
         currentWindow = new net.sf.jsi.Rectangle(gy1, gy1, gy1, gy1);
-        List<Pair<MotifAnchor, MotifAnchor>> yRegions = rTreeHandler.getIntersectingFeatures(chr2.getName(), gy1);
+        List<Pair<GenericLocus, GenericLocus>> yRegions = rTreeHandler.getIntersectingFeatures(chr2.getName(), gy1);
 
         RegionPair rp = RegionPair.generateRegionPair(xRegions.get(0), yRegions.get(0), handler);
         MatrixZoomData zd = zoomDatasForDifferentRegions.get(Matrix.generateKey(rp.xI, rp.yI));
@@ -357,7 +358,7 @@ public double getAverageCount() {
     }
 
 
-    public List<Pair<MotifAnchor, MotifAnchor>> getRTreeHandlerIntersectingFeatures(String name, int g1, int g2) {
+    public List<Pair<GenericLocus, GenericLocus>> getRTreeHandlerIntersectingFeatures(String name, int g1, int g2) {
         return rTreeHandler.getIntersectingFeatures(name, g1, g2);
     }
 }