From e6eebb9c900b7de1f6e97b98386a8d88c41c071b Mon Sep 17 00:00:00 2001 From: ReneEnjilian <enjilianrene@gmail.com> Date: Sun, 14 Jul 2024 11:48:03 +0200 Subject: [PATCH] [SYSTEMDS-3172] Cleanup new MCSC sparse block, new tests Closes #2049. --- .../sysds/runtime/data/SparseBlockCOO.java | 10 +- .../sysds/runtime/data/SparseBlockCSR.java | 8 +- .../runtime/data/SparseBlockFactory.java | 9 +- .../sysds/runtime/data/SparseBlockMCSC.java | 327 ++++++++++++++++-- .../sysds/runtime/data/SparseBlockMCSR.java | 10 +- .../sparse/SparseBlockAlignment.java | 19 +- .../sparse/SparseBlockAppendSort.java | 71 ++-- .../component/sparse/SparseBlockDelete.java | 19 +- .../sparse/SparseBlockGetFirstIndex.java | 128 +++---- .../component/sparse/SparseBlockGetSet.java | 104 +----- .../sparse/SparseBlockIndexRange.java | 74 +--- .../component/sparse/SparseBlockIterator.java | 39 ++- .../sparse/SparseBlockMemEstimate.java | 2 +- .../component/sparse/SparseBlockMerge.java | 167 ++++++++- .../component/sparse/SparseBlockScan.java | 48 +-- .../component/sparse/SparseBlockSize.java | 200 +++++++---- 16 files changed, 776 insertions(+), 459 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java index 1ad2cd57d01..d83a9263dc2 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java @@ -312,15 +312,15 @@ public int size(int r) { public long size(int rl, int ru) { return pos(ru) - pos(rl); } - + @Override public long size(int rl, int ru, int cl, int cu) { long nnz = 0; - for(int i=rl; i<ru; i++) - if( !isEmpty(i) ) { + for(int i = rl; i < ru; i++) + if(!isEmpty(i)) { int start = internPosFIndexGTE(i, cl); - int end = internPosFIndexGTE(i, cu); - nnz += (start!=-1) ? (end-start) : 0; + int end = internPosFIndexLTE(i, cu - 1); + nnz += (start != -1 && end != -1) ? (end - start + 1) : 0; } return nnz; } diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java index 13e844007b4..68cc6c8f864 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java @@ -437,11 +437,11 @@ public long size(int rl, int ru) { @Override public long size(int rl, int ru, int cl, int cu) { long nnz = 0; - for(int i=rl; i<ru; i++) - if( !isEmpty(i) ) { + for(int i = rl; i < ru; i++) + if(!isEmpty(i)) { int start = internPosFIndexGTE(i, cl); - int end = internPosFIndexGTE(i, cu); - nnz += (start!=-1) ? (end-start) : 0; + int end = internPosFIndexLTE(i, cu - 1); + nnz += (start != -1 && end != -1) ? (end - start + 1) : 0; } return nnz; } diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockFactory.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockFactory.java index a6297f8f5b8..106dd36ba9c 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockFactory.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockFactory.java @@ -48,7 +48,12 @@ public static SparseBlock createSparseBlock(SparseBlock.Type type, SparseRow row return ret; } - public static SparseBlock copySparseBlock( SparseBlock.Type type, SparseBlock sblock, boolean forceCopy ) + public static SparseBlock copySparseBlock(SparseBlock.Type type, SparseBlock sblock, boolean forceCopy) { + //Call this method in case 'type' is row format + return copySparseBlock(type, sblock, forceCopy, 1000); // Default clen value + } + + public static SparseBlock copySparseBlock( SparseBlock.Type type, SparseBlock sblock, boolean forceCopy , int clen) { //sanity check for empty inputs if( sblock == null ) @@ -65,7 +70,7 @@ public static SparseBlock copySparseBlock( SparseBlock.Type type, SparseBlock sb case CSR: return new SparseBlockCSR(sblock); case COO: return new SparseBlockCOO(sblock); case DCSR: return new SparseBlockDCSR(sblock); - case MCSC: return new SparseBlockMCSC(sblock); + case MCSC: return new SparseBlockMCSC(sblock, clen); default: throw new RuntimeException("Unexpected sparse block type: "+type.toString()); } diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSC.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSC.java index ec744d60a78..7a12d3e7aaf 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSC.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSC.java @@ -42,13 +42,16 @@ public class SparseBlockMCSC extends SparseBlock { private SparseRow[] _columns = null; private int _clenInferred = -1; + private int _rlen = -1; public SparseBlockMCSC(SparseBlock sblock, int clen) { _clenInferred = clen; + _rlen = sblock.numRows(); initialize(sblock); } public SparseBlockMCSC(SparseBlock sblock) { + _rlen = sblock.numRows(); initialize(sblock); } @@ -153,7 +156,8 @@ else if(columnSize == 1) { } } - public SparseBlockMCSC(SparseRow[] cols, boolean deep) { + public SparseBlockMCSC(SparseRow[] cols, boolean deep, int rlen) { + _rlen = rlen; if(deep) { _columns = new SparseRow[cols.length]; for(int i = 0; i < _columns.length; i++) { @@ -171,7 +175,8 @@ public SparseBlockMCSC(int clen) { } public SparseBlockMCSC(int rlen, int clen) { - this(clen); + _rlen = rlen; + _columns = new SparseRow[clen]; } /** @@ -239,28 +244,49 @@ public long getExactSizeInMemory() { //SparseBlock implementation @Override - public void allocate(int c) { - if(!isAllocated(c)) { + public void allocate(int r) { + for(int i = 0; i < _columns.length; i++) { + if(!isAllocatedCol(i)) + _columns[i] = new SparseRowVector(); + } + } + + public void allocateCol(int c) { + if(!isAllocatedCol(c)) { _columns[c] = new SparseRowVector(); } } @Override - public void allocate(int c, int nnz) { + public void allocate(int r, int nnz) { + allocate(r); + } + + public void allocateCol(int c, int nnz) { if(!isAllocated(c)) { _columns[c] = (nnz == 1) ? new SparseRowScalar() : new SparseRowVector(nnz); } } @Override - public void allocate(int c, int ennz, int maxnnz) { + public void allocate(int r, int ennz, int maxnnz) { + allocate(r); + } + + public void allocateCol(int c, int ennz, int maxnnz) { if(!isAllocated(c)) { _columns[c] = (ennz == 1) ? new SparseRowScalar() : new SparseRowVector(ennz, maxnnz); } } @Override - public void compact(int c) { + public void compact(int r) { + for(int i = 0; i < _columns.length; i++) { + compactCol(i); + } + } + + public void compactCol(int c) { if(isAllocated(c)) { if(_columns[c] instanceof SparseRowVector && _columns[c].size() > SparseBlock.INIT_CAPACITY && _columns[c].size() * SparseBlock.RESIZE_FACTOR1 < ((SparseRowVector) _columns[c]).capacity()) { @@ -272,13 +298,11 @@ else if(_columns[c] instanceof SparseRowScalar) { _columns[c] = null; } } - } @Override public int numRows() { - // this is a column-oriented layout - return 0; + return _rlen; } public int numCols() { @@ -296,7 +320,14 @@ public boolean isContiguous() { } @Override - public boolean isAllocated(int c) { + public boolean isAllocated(int r) { + for(SparseRow col : _columns) + if(col == null) + return false; + return true; + } + + public boolean isAllocatedCol(int c) { return _columns[c] != null; } @@ -319,8 +350,19 @@ public void reset(int ennz, int maxnnz) { } @Override - public void reset(int c, int ennz, int maxnnz) { - if(isAllocated(c)) { + public void reset(int r, int ennz, int maxnnz) { + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i)) { + if(_columns[i] instanceof SparseRowScalar && _columns[i].indexes()[0] == r) + _columns[i].set(r, 0); + else if(_columns[i] instanceof SparseRowVector) + _columns[i].set(r, 0); + } + } + } + + public void resetCol(int c, int ennz, int maxnnz) { + if(isAllocatedCol(c)) { _columns[c].reset(ennz, maxnnz); } } @@ -337,13 +379,33 @@ public long size() { } @Override - public int size(int c) { + public int size(int r) { + int nnz = 0; + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i)) + nnz += (_columns[i].get(r) != 0) ? 1 : 0; + } + return nnz; + } + + public int sizeCol(int c) { //prior check with isEmpty(r) expected return isAllocated(c) ? _columns[c].size() : 0; } @Override - public long size(int cl, int cu) { + public long size(int rl, int ru) { + long nnz = 0; + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i)) { + for(int j = rl; j < ru; j++) + nnz += (_columns[i].get(j) != 0) ? 1 : 0; + } + } + return nnz; + } + + public long sizeCol(int cl, int cu) { long nnz = 0; for(int i = cl; i < cu; i++) { nnz += isAllocated(i) ? _columns[i].size() : 0; @@ -355,17 +417,27 @@ public long size(int cl, int cu) { public long size(int rl, int ru, int cl, int cu) { long nnz = 0; for(int i = cl; i < cu; i++) { - if(!isEmpty(i)) { - int start = posFIndexGTE(rl, i); - int end = posFIndexGTE(ru, i); - nnz += (start != -1) ? (end - start) : 0; + if(!isEmptyCol(i)) { + int start = posFIndexGTECol(rl, i); + int end = posFIndexLTECol(ru - 1, i); + nnz += (start != -1 && end != -1) ? (end - start + 1) : 0; } } return nnz; } @Override - public boolean isEmpty(int c) { + public boolean isEmpty(int r) { + for(int i = 0; i < _columns.length; i++) { + if(!isAllocatedCol(i)) + continue; + else if(_columns[i].get(r) != 0) + return false; + } + return true; + } + + public boolean isEmptyCol(int c) { return _columns[c] == null || _columns[c].isEmpty(); } @@ -413,26 +485,54 @@ public boolean checkValidity(int rlen, int clen, long nnz, boolean strict) { } @Override - public int[] indexes(int c) { + public int[] indexes(int r) { + //prior check with isEmpty(r) expected + int nnz = size(r); + int[] idx = new int[nnz]; + int index = 0; + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i) && _columns[i].get(r) != 0) { + idx[index] = i; + index++; + } + } + return idx; + } + + public int[] indexesCol(int c) { //prior check with isEmpty(c) expected return _columns[c].indexes(); } @Override - public double[] values(int c) { + public double[] values(int r) { + //prior check with isEmpty(r) expected + int nnz = size(r); + double[] vals = new double[nnz]; + int index = 0; + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i) && _columns[i].get(r) != 0) { + vals[index] = _columns[i].get(r); + index++; + } + } + return vals; + } + + public double[] valuesCol(int c) { //prior check with isEmpty(c) expected return _columns[c].values(); } @Override - public int pos(int c) { - //arrays per column (always start 0) + public int pos(int r) { + //arrays per row (always start 0) return 0; } @Override public boolean set(int r, int c, double v) { - if(!isAllocated(c)) { + if(!isAllocatedCol(c)) { _columns[c] = new SparseRowScalar(); } else if(_columns[c] instanceof SparseRowScalar && !_columns[c].isEmpty()) { @@ -442,9 +542,17 @@ else if(_columns[c] instanceof SparseRowScalar && !_columns[c].isEmpty()) { } @Override - public void set(int c, SparseRow col, boolean deep) { + public void set(int r, SparseRow row, boolean deep) { + reset(r, 1, 1); + int nnz = row.size(); + for(int i = 0; i < nnz; i++) { + set(r, row.indexes()[i], row.values()[i]); + } + } + + public void setCol(int c, SparseRow col, boolean deep) { //copy values into existing column to avoid allocation - if(isAllocated(c) && _columns[c] instanceof SparseRowVector && + if(isAllocatedCol(c) && _columns[c] instanceof SparseRowVector && ((SparseRowVector) _columns[c]).capacity() >= col.size() && deep) { ((SparseRowVector) _columns[c]).copy(col); //set new sparse column (incl allocation if required) @@ -456,7 +564,7 @@ public void set(int c, SparseRow col, boolean deep) { @Override public boolean add(int r, int c, double v) { - if(!isAllocated(c)) { + if(!isAllocatedCol(c)) { _columns[c] = new SparseRowScalar(); } else if(_columns[c] instanceof SparseRowScalar && !_columns[c].isEmpty()) { @@ -485,8 +593,16 @@ else if(_columns[c] == null) { } @Override - public void setIndexRange(int c, int rl, int ru, double[] v, int vix, int vlen) { - if(!isAllocated(c)) { + public void setIndexRange(int r, int cl, int cu, double[] v, int vix, int vlen) { + int idx = vix; + for(int i = cl; i < cu; i++) { + set(r, i, v[idx]); + idx++; + } + } + + public void setIndexRangeCol(int c, int rl, int ru, double[] v, int vix, int vlen) { + if(!isAllocatedCol(c)) { _columns[c] = new SparseRowVector(); } else if(_columns[c] instanceof SparseRowScalar) { @@ -496,8 +612,14 @@ else if(_columns[c] instanceof SparseRowScalar) { } @Override - public void setIndexRange(int c, int rl, int ru, double[] v, int[] vix, int vpos, int vlen) { - if(!isAllocated(c)) { + public void setIndexRange(int r, int cl, int cu, double[] v, int[] vix, int vpos, int vlen) { + for(int i = vpos; i < (vpos + vlen); i++) { + set(r, vix[i], v[i]); + } + } + + public void setIndexRangeCol(int c, int rl, int ru, double[] v, int[] vix, int vpos, int vlen) { + if(!isAllocatedCol(c)) { _columns[c] = new SparseRowVector(); } else if(_columns[c] instanceof SparseRowScalar) { @@ -508,7 +630,18 @@ else if(_columns[c] instanceof SparseRowScalar) { } @Override - public void deleteIndexRange(int c, int rl, int ru) { + public void deleteIndexRange(int r, int cl, int cu) { + for(int i = cl; i < cu; i++) { + if(isAllocatedCol(i)) { + if(_columns[i] instanceof SparseRowScalar && _columns[i].indexes()[0] == r) + _columns[i].set(r, 0); + else if(_columns[i] instanceof SparseRowVector) + _columns[i].set(r, 0); + } + } + } + + public void deleteIndexRangeCol(int c, int rl, int ru) { //prior check with isEmpty(c) expected //different sparse row semantics: upper bound inclusive if(_columns[c] instanceof SparseRowScalar) { @@ -527,26 +660,48 @@ public void sort() { } @Override - public void sort(int c) { + public void sort(int r) { + //prior check with isEmpty(c) expected + sort(); + } + + public void sortCol(int c) { //prior check with isEmpty(c) expected _columns[c].sort(); } @Override public double get(int r, int c) { - if(!isAllocated(c)) { + if(!isAllocatedCol(c)) { return 0; } return _columns[c].get(r); } @Override - public SparseRow get(int c) { + public SparseRow get(int r) { + SparseRow row = (size(r) == 1) ? new SparseRowScalar() : new SparseRowVector(size(r)); + double v = 0; + for(int i = 0; i < _columns.length; i++) { + v = get(r, i); + if(v != 0) + row.set(i, v); + } + return row; + } + + public SparseRow getCol(int c) { return _columns[c]; } @Override public int posFIndexLTE(int r, int c) { + //prior check with isEmpty(r) expected + SparseRow row = get(r); + return ((SparseRowVector) row).searchIndexesFirstLTE(c); + } + + public int posFIndexLTECol(int r, int c) { //prior check with isEmpty(c) expected if(_columns[c] instanceof SparseRowScalar) { _columns[c] = new SparseRowVector(_columns[c]); @@ -556,17 +711,49 @@ public int posFIndexLTE(int r, int c) { @Override public int posFIndexGTE(int r, int c) { + SparseRow row = get(r); + return row.searchIndexesFirstGTE(c); + } + + public int posFIndexGTECol(int r, int c) { return _columns[c].searchIndexesFirstGTE(r); } @Override public int posFIndexGT(int r, int c) { + SparseRow row = get(r); + return row.searchIndexesFirstGT(c); + } + + public int posFIndexGTCol(int r, int c) { return _columns[c].searchIndexesFirstGT(r); } @Override public Iterator<Integer> getNonEmptyRowsIterator(int rl, int ru) { - throw new UnsupportedOperationException("Non-empty rows iterator is not supported in column layouts."); + return new NonEmptyRowsIteratorMCSC(rl, ru); + } + + public class NonEmptyRowsIteratorMCSC implements Iterator<Integer> { + private int _rpos; + private final int _ru; + + public NonEmptyRowsIteratorMCSC(int rl, int ru) { + _rpos = rl; + _ru = ru; + } + + @Override + public boolean hasNext() { + while(_rpos < _ru && isEmpty(_rpos)) + _rpos++; + return _rpos < _ru; + } + + @Override + public Integer next() { + return _rpos++; + } } @Override @@ -580,7 +767,7 @@ public String toString() { sb.append("\n"); final int colDigits = (int) Math.max(Math.ceil(Math.log10(nCol)), 1); for(int i = 0; i < nCol; i++) { - if(isEmpty(i)) + if(isEmptyCol(i)) continue; sb.append(String.format("%0" + colDigits + "d %s\n", i, _columns[i].toString())); } @@ -588,7 +775,71 @@ public String toString() { return sb.toString(); } + /** + * Helper function for MCSC + * + * @return the underlying array of columns {@link SparseRow} + */ public SparseRow[] getCols() { return _columns; } + + /** + * Helper function for MCSC + * + * @return the corresponding array of rows {@link SparseRow} + */ + + public SparseRow[] getRows() { + SparseRow[] rows = new SparseRow[numRows()]; + for(int i = 0; i < numRows(); i++) { + rows[i] = get(i); + } + return rows; + } + + public Iterator<Integer> getNonEmptyColumnsIterator(int cl, int cu) { + return new NonEmptyColumnsIteratorMCSC(cl, cu); + } + + public class NonEmptyColumnsIteratorMCSC implements Iterator<Integer> { + private int _cpos; + private final int _cu; + + public NonEmptyColumnsIteratorMCSC(int cl, int cu) { + _cpos = cl; + _cu = cu; + } + + @Override + public boolean hasNext() { + while(_cpos < _cu && isEmptyCol(_cpos)) { + _cpos++; + } + return _cpos < _cu; + } + + @Override + public Integer next() { + return _cpos++; + } + + } + + @SuppressWarnings("unused") + private class SparseNonEmptyColumnIterable implements Iterable<Integer> { + private final int _cl; //column lower + private final int _cu; //column upper + + protected SparseNonEmptyColumnIterable(int cl, int cu) { + _cl = cl; + _cu = cu; + } + + @Override + public Iterator<Integer> iterator() { + //use specialized non-empty row iterators of sparse blocks + return getNonEmptyColumnsIterator(_cl, _cu); + } + } } diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java index 52b5d2e338a..d232a0041a6 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java @@ -305,15 +305,15 @@ public long size(int rl, int ru) { ret += isAllocated(i) ? _rows[i].size() : 0; return ret; } - + @Override public long size(int rl, int ru, int cl, int cu) { long nnz = 0; - for(int i=rl; i<ru; i++) - if( !isEmpty(i) ) { + for(int i = rl; i < ru; i++) + if(!isEmpty(i)) { int start = posFIndexGTE(i, cl); - int end = posFIndexGTE(i, cu); - nnz += (start!=-1) ? (end-start) : 0; + int end = posFIndexLTE(i, cu - 1); + nnz += (start != -1 && end != -1) ? (end - start + 1) : 0; } return nnz; } diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockAlignment.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockAlignment.java index f6d2b73b692..41caea2e379 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockAlignment.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockAlignment.java @@ -182,6 +182,21 @@ public void testSparseBlockDCSR3Neg() { runSparseBlockScanTest(SparseBlock.Type.DCSR, sparsity3, false); } + @Test + public void testSparseBlockMCSC1Neg() { + runSparseBlockScanTest(SparseBlock.Type.MCSC, sparsity1, false); + } + + @Test + public void testSparseBlockMCSC2Neg() { + runSparseBlockScanTest(SparseBlock.Type.MCSC, sparsity2, false); + } + + @Test + public void testSparseBlockMCSC3Neg() { + runSparseBlockScanTest(SparseBlock.Type.MCSC, sparsity3, false); + } + private void runSparseBlockScanTest( SparseBlock.Type btype, double sparsity, boolean positive) { try @@ -192,10 +207,10 @@ private void runSparseBlockScanTest( SparseBlock.Type btype, double sparsity, bo //init sparse block MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A); SparseBlock srtmp = mbtmp.getSparseBlock(); - SparseBlock sblock = SparseBlockFactory.copySparseBlock(btype, srtmp, true); + SparseBlock sblock = SparseBlockFactory.copySparseBlock(btype, srtmp, true, cols); //init second sparse block and deep copy - SparseBlock sblock2 = SparseBlockFactory.copySparseBlock(btype, sblock, true); + SparseBlock sblock2 = SparseBlockFactory.copySparseBlock(btype, sblock, true, cols); //modify second block if necessary if( !positive ) { diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockAppendSort.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockAppendSort.java index 8c42af17af6..37bdd4fb874 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockAppendSort.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockAppendSort.java @@ -230,68 +230,45 @@ private void runSparseBlockAppendSortTest( SparseBlock.Type btype, double sparsi for( int j=0; j<cols; j++ ) sblock.append(i, j, A[i][j]); } - else if( itype == InitType.RAND_SET ) { + else if(itype == InitType.RAND_SET) { LongLongDoubleHashMap map = new LongLongDoubleHashMap(); - for( int i=0; i<rows; i++ ) - for( int j=0; j<cols; j++ ) + for(int i = 0; i < rows; i++) + for(int j = 0; j < cols; j++) map.addValue(i, j, A[i][j]); Iterator<ADoubleEntry> iter = map.getIterator(); - while( iter.hasNext() ) { //random hash order + while(iter.hasNext()) { //random hash order ADoubleEntry e = iter.next(); - sblock.append((int)e.getKey1(), (int)e.getKey2(), e.value); + sblock.append((int) e.getKey1(), (int) e.getKey2(), e.value); } - } - + } + //sort appended values sblock.sort(); - + //check for correct number of non-zeros - int[] rnnz = new int[rows]; - int nnz = 0; - int[] cnnz = new int[cols]; - for( int i=0; i<rows; i++ ) { - for( int j=0; j<cols; j++ ) { - cnnz[j] += (A[i][j] != 0) ? 1 : 0; + int[] rnnz = new int[rows]; int nnz = 0; + for(int i = 0; i < rows; i++) { + for(int j = 0; j < cols; j++) rnnz[i] += (A[i][j] != 0) ? 1 : 0; - } nnz += rnnz[i]; } - if( nnz != sblock.size() ) - Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz); + + if(nnz != sblock.size()) + Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz); //check correct isEmpty return - if(sblock instanceof SparseBlockMCSC) { - for(int i = 0; i < cols; i++) - if(sblock.isEmpty(i) != (cnnz[i] == 0)) - Assert.fail("Wrong isEmpty(column) result for row nnz: " + cnnz[i]); - } - else { - for(int i = 0; i < rows; i++) - if(sblock.isEmpty(i) != (rnnz[i] == 0)) - Assert.fail("Wrong isEmpty(row) result for row nnz: " + rnnz[i]); - } + for(int i = 0; i < rows; i++) + if(sblock.isEmpty(i) != (rnnz[i] == 0)) + Assert.fail("Wrong isEmpty(row) result for row nnz: " + rnnz[i]); //check correct values - if(sblock instanceof SparseBlockMCSC) { - for(int i = 0; i < cols; i++) { - if(sblock.isEmpty(i)) continue; - for(int j = 0; j < rows; j++) { - double tmp = sblock.get(j, i); - if(tmp != A[j][i]) - Assert.fail("Wrong get value for cell (" + i + "," + j + "): " + tmp + ", expected: " + - A[i][j]); - } - } - } - else { - for(int i = 0; i < rows; i++) { - if(sblock.isEmpty(i)) continue; - for(int j = 0; j < cols; j++) { - double tmp = sblock.get(i, j); - if(tmp != A[i][j]) - Assert.fail("Wrong get value for cell (" + i + "," + j + "): " + tmp + ", expected: " + - A[i][j]); - } + for(int i = 0; i < rows; i++) { + if(sblock.isEmpty(i)) + continue; + for(int j = 0; j < cols; j++) { + double tmp = sblock.get(i, j); + if(tmp != A[i][j]) + Assert.fail("Wrong get value for cell (" + i + "," + j + "): " + tmp + ", expected: " + A[i][j]); } } } diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockDelete.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockDelete.java index 9862659779f..7ac63c0cee6 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockDelete.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockDelete.java @@ -111,6 +111,21 @@ public void testSparseBlockDCSR2() { public void testSparseBlockDCSR3() { runSparseBlockDeleteTest(SparseBlock.Type.DCSR, sparsity3); } + + @Test + public void testSparseBlockMCSC1() { + runSparseBlockDeleteTest(SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testSparseBlockMCSC2() { + runSparseBlockDeleteTest(SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testSparseBlockMCSC3() { + runSparseBlockDeleteTest(SparseBlock.Type.MCSC, sparsity3); + } private void runSparseBlockDeleteTest( SparseBlock.Type btype, double sparsity) { @@ -122,7 +137,7 @@ private void runSparseBlockDeleteTest( SparseBlock.Type btype, double sparsity) //init sparse block MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A); SparseBlock srtmp = mbtmp.getSparseBlock(); - SparseBlock sblock = SparseBlockFactory.copySparseBlock(btype, srtmp, true); + SparseBlock sblock = SparseBlockFactory.copySparseBlock(btype, srtmp, true, cols); //delete range per row via set for( int i=0; i<rows; i++ ) @@ -163,4 +178,4 @@ private void runSparseBlockDeleteTest( SparseBlock.Type btype, double sparsity) throw new RuntimeException(ex); } } -} \ No newline at end of file +} diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetFirstIndex.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetFirstIndex.java index 1ad84df759a..5816ba6b26d 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetFirstIndex.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetFirstIndex.java @@ -287,80 +287,69 @@ private void runSparseBlockGetFirstIndexTest( SparseBlock.Type btype, double spa try { //data generation - double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 3456); - + double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 3456); + //init sparse block SparseBlock sblock = null; MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A); SparseBlock srtmp = mbtmp.getSparseBlock(); - switch( btype ) { - case MCSR: sblock = new SparseBlockMCSR(srtmp); break; - case CSR: sblock = new SparseBlockCSR(srtmp); break; - case COO: sblock = new SparseBlockCOO(srtmp); break; - case DCSR: sblock = new SparseBlockDCSR(srtmp); break; - case MCSC: sblock = new SparseBlockMCSC(srtmp, cols); break; + switch(btype) { + case MCSR: + sblock = new SparseBlockMCSR(srtmp); + break; + case CSR: + sblock = new SparseBlockCSR(srtmp); + break; + case COO: + sblock = new SparseBlockCOO(srtmp); + break; + case DCSR: + sblock = new SparseBlockDCSR(srtmp); + break; + case MCSC: + sblock = new SparseBlockMCSC(srtmp, cols); + break; } - + //check for correct number of non-zeros - int[] rnnz = new int[rows]; int nnz = 0; - int[] cnnz =new int[cols]; - for( int i=0; i<rows; i++ ) { - for( int j=0; j<cols; j++ ) { - cnnz[j] += (A[i][j] != 0) ? 1 : 0; + int[] rnnz = new int[rows]; + int nnz = 0; + for(int i = 0; i < rows; i++) { + for(int j = 0; j < cols; j++) rnnz[i] += (A[i][j] != 0) ? 1 : 0; - } nnz += rnnz[i]; } - if( nnz != sblock.size() ) - Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz); + + if(nnz != sblock.size()) + Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz); //check correct isEmpty return - if(sblock instanceof SparseBlockMCSC) { - for(int i = 0; i < cols; i++) - if(sblock.isEmpty(i) != (cnnz[i] == 0)) - Assert.fail("Wrong isEmpty(col) result for row nnz: " + cnnz[i]); - } - else { - for(int i = 0; i < rows; i++) - if(sblock.isEmpty(i) != (rnnz[i] == 0)) - Assert.fail("Wrong isEmpty(row) result for row nnz: " + rnnz[i]); - } + for(int i = 0; i < rows; i++) + if(sblock.isEmpty(i) != (rnnz[i] == 0)) + Assert.fail("Wrong isEmpty(row) result for row nnz: " + rnnz[i]); //check correct index values - if(sblock instanceof SparseBlockMCSC){ - for (int i = 0; i < cols; i++) { - int ix = getFirstIxCol(A, i, i, itype); - int sixpos = -1; - switch (itype) { - case GT: sixpos = sblock.posFIndexGT(i, i); break; - case GTE: sixpos = sblock.posFIndexGTE(i, i); break; - case LTE: sixpos = sblock.posFIndexLTE(i, i); break; - } - int six = (sixpos >= 0) ? - sblock.indexes(i)[sblock.pos(i) + sixpos] : -1; - if (six != ix) { - Assert.fail("Wrong index returned by index probe (" + - itype.toString() + "," + i + "): " + six + ", expected: " + ix); - } + for(int i = 0; i < rows; i++) { + int ix = getFirstIx(A, i, i, itype); + int sixpos = -1; + switch(itype) { + case GT: + sixpos = sblock.posFIndexGT(i, i); + break; + case GTE: + sixpos = sblock.posFIndexGTE(i, i); + break; + case LTE: + sixpos = sblock.posFIndexLTE(i, i); + break; } - } - else{ - for( int i=0; i<rows; i++ ) { - int ix = getFirstIx(A, i, i, itype); - int sixpos = -1; - switch( itype ) { - case GT: sixpos = sblock.posFIndexGT(i, i); break; - case GTE: sixpos = sblock.posFIndexGTE(i, i); break; - case LTE: sixpos = sblock.posFIndexLTE(i, i); break; - } - int six = (sixpos>=0) ? - sblock.indexes(i)[sblock.pos(i)+sixpos] : -1; - if( six != ix ) { - Assert.fail("Wrong index returned by index probe ("+ - itype.toString()+","+i+"): "+six+", expected: "+ix); - } + int six = (sixpos >= 0) ? sblock.indexes(i)[sblock.pos(i) + sixpos] : -1; + if(six != ix) { + Assert.fail("Wrong index returned by index probe (" + itype.toString() + "," + i + "): " + + six + ", expected: " + ix); } } + } catch(Exception ex) { ex.printStackTrace(); @@ -391,27 +380,4 @@ else if( type==IndexType.LTE ) { return -1; } - private static int getFirstIxCol(double[][] A, int rix, int cix, IndexType type) { - if(type == IndexType.GT) { - for(int j = rix + 1; j < rows; j++) - if(A[j][cix] != 0) - return j; - return -1; - } - else if(type == IndexType.GTE) { - for(int j = rix; j < rows; j++) - if(A[j][cix] != 0) - return j; - return -1; - } - else if(type == IndexType.LTE) { - for(int j = rix; j >= 0; j--) - if(A[j][cix] != 0) - return j; - return -1; - } - - return -1; - } - } diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetSet.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetSet.java index bb284d34a47..2f357d3c8ab 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetSet.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetSet.java @@ -244,47 +244,47 @@ public void testSparseBlockDCSR3Rand() { @Test public void testSparseBlockMCSC1Bulk() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity1, InitType.BULK); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity1, InitType.BULK); } @Test public void testSparseBlockMCSC2Bulk() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity2, InitType.BULK); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity2, InitType.BULK); } @Test public void testSparseBlockMCSC3Bulk() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity3, InitType.BULK); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity3, InitType.BULK); } @Test public void testSparseBlockMCSC1Seq() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity1, InitType.SEQ_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity1, InitType.SEQ_SET); } @Test public void testSparseBlockMCSC2Seq() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity2, InitType.SEQ_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity2, InitType.SEQ_SET); } @Test public void testSparseBlockMCSC3Seq() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity3, InitType.SEQ_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity3, InitType.SEQ_SET); } @Test public void testSparseBlockMCSC1Rand() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity1, InitType.RAND_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity1, InitType.RAND_SET); } @Test public void testSparseBlockMCSC2Rand() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity2, InitType.RAND_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity2, InitType.RAND_SET); } @Test public void testSparseBlockMCSC3Rand() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity3, InitType.RAND_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity3, InitType.RAND_SET); } private void runSparseBlockGetSetTest( SparseBlock.Type btype, double sparsity, InitType itype) @@ -310,10 +310,6 @@ else if( itype == InitType.SEQ_SET || itype == InitType.RAND_SET ) { case MCSC: sblock = new SparseBlockMCSC(rows, cols); break; } - if(sblock instanceof SparseBlockMCSC){ - - } - if(itype == InitType.SEQ_SET) { for( int i=0; i<rows; i++ ) for( int j=0; j<cols; j++ ) @@ -368,86 +364,4 @@ else if( itype == InitType.RAND_SET ) { } } - @SuppressWarnings("incomplete-switch") - private void runSparseBlockGetSetColumnTest(SparseBlock.Type btype, double sparsity, InitType itype) - { - try { - //data generation - double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 7654321); - - //init sparse block - SparseBlockMCSC sblock = null; - if(itype == InitType.BULK) { - MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A); - SparseBlock srtmp = mbtmp.getSparseBlock(); - switch(btype) { - case MCSC: - sblock = new SparseBlockMCSC(srtmp, cols); - break; - } - } - else if(itype == InitType.SEQ_SET || itype == InitType.RAND_SET) { - switch(btype) { - case MCSC: - sblock = new SparseBlockMCSC(rows, cols); - break; - } - - if(itype == InitType.SEQ_SET) { - for(int i = 0; i < cols; i++) - for(int j = 0; j < rows; j++) - sblock.append(j, i, A[j][i]); - } - else if(itype == InitType.RAND_SET) { - LongLongDoubleHashMap map = new LongLongDoubleHashMap(); - for(int i = 0; i < cols; i++) - for(int j = 0; j < rows; j++) - map.addValue(j, i, A[j][i]); - Iterator<ADoubleEntry> iter = map.getIterator(); - while(iter.hasNext()) { //random hash order - ADoubleEntry e = iter.next(); - int r = (int) e.getKey1(); - int c = (int) e.getKey2(); - sblock.set(r, c, e.value); - } - } - } - - //check basic meta data - if(sblock.numCols() != cols) - Assert.fail("Wrong number of cols: " + sblock.numCols() + ", expected: " + cols); - - //check for correct number of non-zeros - int[] cnnz = new int[cols]; - int nnz = 0; - for(int i = 0; i < cols; i++) { - for(int j = 0; j < rows; j++) - cnnz[i] += (A[j][i] != 0) ? 1 : 0; - nnz += cnnz[i]; - } - if(nnz != sblock.size()) - Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz); - - //check correct isEmpty return - for(int i = 0; i < cols; i++) - if(sblock.isEmpty(i) != (cnnz[i] == 0)) - Assert.fail("Wrong isEmpty(col) result for row nnz: " + cnnz[i] + "(column: " + i + ")"); - - //check correct values - for(int i = 0; i < cols; i++) - if(!sblock.isEmpty(i)) - for(int j = 0; j < rows; j++) { - double tmp = sblock.get(j, i); - if(tmp != A[j][i]) - Assert.fail( - "Wrong get value for cell (" + i + "," + j + "): " + tmp + ", expected: " + A[j][i]); - } - - } - catch(Exception ex) { - ex.printStackTrace(); - throw new RuntimeException(ex); - } - } - } diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIndexRange.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIndexRange.java index 4a87f0d129e..d51f9d1f6da 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIndexRange.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIndexRange.java @@ -43,8 +43,6 @@ public class SparseBlockIndexRange extends AutomatedTestBase private final static int cols = 549; private final static int cl = 245; private final static int cu = 425; - private final static int rl = 245; - private final static int ru = 525; private final static double sparsity1 = 0.12; private final static double sparsity2 = 0.22; private final static double sparsity3 = 0.32; @@ -181,32 +179,32 @@ public void testSparseBlockDCSR3Insert() { @Test public void testSparseBlockMCSC1Delete() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity1, UpdateType.DELETE); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity1, UpdateType.DELETE); } @Test public void testSparseBlockMCSC2Delete() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity2, UpdateType.DELETE); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity2, UpdateType.DELETE); } @Test public void testSparseBlockMCSC3Delete() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity3, UpdateType.DELETE); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity3, UpdateType.DELETE); } @Test public void testSparseBlockMCSC1Insert() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity1, UpdateType.INSERT); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity1, UpdateType.INSERT); } @Test public void testSparseBlockMCSC2Insert() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity2, UpdateType.INSERT); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity2, UpdateType.INSERT); } @Test public void testSparseBlockMCSC3Insert() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity3, UpdateType.INSERT); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity3, UpdateType.INSERT); } private void runSparseBlockIndexRangeTest( SparseBlock.Type btype, double sparsity, UpdateType utype) @@ -278,64 +276,4 @@ else if( utype == UpdateType.INSERT ) { } } - @SuppressWarnings("incomplete-switch") - private void runSparseBlockIndexRangeColumnTest(SparseBlock.Type btype, double sparsity, UpdateType utype) { - try { - //data generation - double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 456); - - //init sparse block - SparseBlock sblock = null; - MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A); - SparseBlock srtmp = mbtmp.getSparseBlock(); - switch(btype) { - case MCSC: - sblock = new SparseBlockMCSC(srtmp, cols); - break; - } - - //delete range per row via set - if(utype == UpdateType.DELETE) { - for(int i = 0; i < cols; i++) { - sblock.deleteIndexRange(i, rl, ru); - for(int j = rl; j < ru; j++) { - A[j][i] = 0; // Fill column-wise with zeros - } - } - } - else if(utype == UpdateType.INSERT) { - double[] vals = new double[ru - rl]; - for(int j = rl; j < ru; j++) - vals[j - rl] = j; - for(int i = 0; i < cols; i++) { - sblock.setIndexRange(i, rl, ru, vals, 0, ru - rl); - for(int j = rl; j < ru; j++) { - A[j][i] = vals[j - rl]; // Update the matrix column-wise - } - } - } - - //check for correct number of non-zeros - int[] cnnz = new int[cols]; - int nnz = 0; - for(int i = 0; i < cols; i++) { - for(int j = 0; j < rows; j++) - cnnz[i] += (A[j][i] != 0) ? 1 : 0; - nnz += cnnz[i]; - } - if(nnz != sblock.size()) - Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz); - - //check correct isEmpty return - for(int i = 0; i < cols; i++) - if(sblock.isEmpty(i) != (cnnz[i] == 0)) - Assert.fail("Wrong isEmpty(col) result for row nnz: " + cnnz[i]); - - //TODO: Add 'check correct values', requires Iterator - } - catch(Exception ex) { - ex.printStackTrace(); - throw new RuntimeException(ex); - } - } } diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIterator.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIterator.java index 0039d5ac1ea..736f981025d 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIterator.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIterator.java @@ -172,6 +172,36 @@ public void testSparseBlockDCSR3Partial() { runSparseBlockIteratorTest(SparseBlock.Type.DCSR, sparsity3, true); } + @Test + public void testSparseBlockMCSC1Full() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity1, false); + } + + @Test + public void testSparseBlockMCSC2Full() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity2, false); + } + + @Test + public void testSparseBlockMCSC3Full() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity3, false); + } + + @Test + public void testSparseBlockMCSC1Partial() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity1, true); + } + + @Test + public void testSparseBlockMCSC2Partial() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity2, true); + } + + @Test + public void testSparseBlockMCSC3Partial() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity3, true); + } + private void runSparseBlockIteratorTest(SparseBlock.Type btype, double sparsity, boolean partial) { try { //data generation @@ -181,7 +211,7 @@ private void runSparseBlockIteratorTest(SparseBlock.Type btype, double sparsity, MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A); SparseBlock srtmp = mbtmp.getSparseBlock(); SparseBlock sblock = SparseBlockFactory.copySparseBlock(btype, srtmp, true); - + //check for correct number of non-zeros int[] rnnz = new int[rows]; int nnz = 0; @@ -215,9 +245,9 @@ private void runSparseBlockIteratorTest(SparseBlock.Type btype, double sparsity, // check iterator over non-zero rows List<Integer> manualNonZeroRows = new ArrayList<>(); List<Integer> iteratorNonZeroRows = new ArrayList<>(); - Iterator<Integer> iterRows = !partial ? - sblock.getNonEmptyRowsIterator(0, rows) : - sblock.getNonEmptyRowsIterator(rl, rows); + Iterator<Integer> iterRows = !partial + ? sblock.getNonEmptyRowsIterator(0, rows) + : sblock.getNonEmptyRowsIterator(rl, rows); for(int i = rl; i < rows; i++) if(!sblock.isEmpty(i)) @@ -230,6 +260,7 @@ private void runSparseBlockIteratorTest(SparseBlock.Type btype, double sparsity, if(!manualNonZeroRows.equals(iteratorNonZeroRows)) { Assert.fail("Verification of iterator over non-zero rows failed."); } + } catch(Exception ex) { ex.printStackTrace(); diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMemEstimate.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMemEstimate.java index 36dc077fb06..1c0c2aa7c5c 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMemEstimate.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMemEstimate.java @@ -98,7 +98,7 @@ private static void runSparseBlockMemoryTest( double sparsity) } else { //ultra-sparse (pref COO) if( memMCSC < memCOO ) - Assert.fail("SparseBlockMCS memory estimate smaller than SparseBlockCOO estimate."); + Assert.fail("SparseBlockMCSC memory estimate smaller than SparseBlockCOO estimate."); if( memMCSR < memCOO ) Assert.fail("SparseBlockMCSR memory estimate smaller than SparseBlockCOO estimate."); if( memCSR < memCOO ) diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMerge.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMerge.java index e5681e294e9..7c061eaf279 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMerge.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMerge.java @@ -122,6 +122,26 @@ public void testMergeMCSR_DCSR_2() { public void testMergeMCSR_DCSR_3() { runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.DCSR, sparsity3); } + + @Test + public void testMergeMCSR_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeMCSR_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeMCSR_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeMCSR_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.MCSC, sparsity3); + } @Test public void testMergeCSR_CSR_0() { @@ -202,6 +222,26 @@ public void testMergeCSR_COO_2() { public void testMergeCSR_COO_3() { runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.COO, sparsity3); } + + @Test + public void testMergeCSR_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeCSR_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeCSR_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeCSR_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.MCSC, sparsity3); + } @Test public void testMergeCOO_COO_0() { @@ -283,6 +323,26 @@ public void testMergeCOO_DCSR_3() { runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.DCSR, sparsity3); } + @Test + public void testMergeCOO_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeCOO_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeCOO_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeCOO_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.MCSC, sparsity3); + } + @Test public void testMergeDCSR_DCSR_0() { runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.DCSR, sparsity0); @@ -363,7 +423,108 @@ public void testMergeDCSR_COO_3() { runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.COO, sparsity3); } - + @Test + public void testMergeDCSR_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeDCSR_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeDCSR_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeDCSR_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.MCSC, sparsity3); + } + + @Test + public void testMergeMCSC_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeMCSC_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeMCSC_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeMCSC_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSC, sparsity3); + } + + @Test + public void testMergeMCSC_CSR_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.CSR, sparsity0); + } + + @Test + public void testMergeMCSC_CSR_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.CSR, sparsity1); + } + + @Test + public void testMergeMCSC_CSR_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.CSR, sparsity2); + } + + @Test + public void testMergeMCSC_CSR_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.CSR, sparsity3); + } + + @Test + public void testMergeMCSC_MCSR_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSR, sparsity0); + } + + @Test + public void testMergeMCSC_MCSR_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSR, sparsity1); + } + + @Test + public void testMergeMCSC_MCSR_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSR, sparsity2); + } + + @Test + public void testMergeMCSC_MCSR_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSR, sparsity3); + } + + @Test + public void testMergeMCSC_COO_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.COO, sparsity0); + } + + @Test + public void testMergeMCSC_COO_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.COO, sparsity1); + } + + @Test + public void testMergeMCSC_COO_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.COO, sparsity2); + } + + @Test + public void testMergeMCSC_COO_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.COO, sparsity3); + } + + + private void runSparseBlockMergeTest( SparseBlock.Type btype1, SparseBlock.Type btype2, double sparsity) { try @@ -387,8 +548,8 @@ private void runSparseBlockMergeTest( SparseBlock.Type btype1, SparseBlock.Type MatrixBlock mb1 = DataConverter.convertToMatrixBlock(B1); MatrixBlock mb2 = DataConverter.convertToMatrixBlock(B2); long nnz = mb1.getNonZeros() + mb2.getNonZeros(); - mb1.setSparseBlock(SparseBlockFactory.copySparseBlock(btype1, mb1.getSparseBlock(), false)); - mb2.setSparseBlock(SparseBlockFactory.copySparseBlock(btype2, mb2.getSparseBlock(), false)); + mb1.setSparseBlock(SparseBlockFactory.copySparseBlock(btype1, mb1.getSparseBlock(), false, cols)); + mb2.setSparseBlock(SparseBlockFactory.copySparseBlock(btype2, mb2.getSparseBlock(), false, cols)); //execute merge mb1 = mb1.merge(mb2, false); diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockScan.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockScan.java index 98fe32792ed..8163b49a6ef 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockScan.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockScan.java @@ -142,48 +142,32 @@ private void runSparseBlockScanTest( SparseBlock.Type btype, double sparsity) //check for correct number of non-zeros int[] rnnz = new int[rows]; int nnz = 0; - int[] cnnz = new int[cols]; - for( int i=0; i<rows; i++ ) { - for( int j=0; j<cols; j++ ) { + for(int i = 0; i < rows; i++) { + for(int j = 0; j < cols; j++) rnnz[i] += (A[i][j] != 0) ? 1 : 0; - cnnz[j] += (A[i][j] != 0) ? 1 : 0; - } nnz += rnnz[i]; } - if( nnz != sblock.size() ) - Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz); - + + if(nnz != sblock.size()) + Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz); + //check correct isEmpty return - if(sblock instanceof SparseBlockMCSC) { - for(int i = 0; i < cols; i++) - if(sblock.isEmpty(i) != (cnnz[i] == 0)) - Assert.fail("Wrong isEmpty(col) result for column nnz: " + cnnz[i]); - } - else { - for(int i = 0; i < rows; i++) - if(sblock.isEmpty(i) != (rnnz[i] == 0)) - Assert.fail("Wrong isEmpty(row) result for row nnz: " + rnnz[i]); - } - + for(int i = 0; i < rows; i++) + if(sblock.isEmpty(i) != (rnnz[i] == 0)) + Assert.fail("Wrong isEmpty(row) result for row nnz: " + rnnz[i]); + //check correct values - int limit = rows; - if(sblock instanceof SparseBlockMCSC) - limit = cols; int count = 0; - for( int i=0; i<limit; i++) { + for(int i = 0; i < rows; i++) { int alen = sblock.size(i); int apos = sblock.pos(i); int[] aix = sblock.indexes(i); double[] avals = sblock.values(i); - for( int j=0; j<alen; j++ ) { - if(sblock instanceof SparseBlockMCSC){ - if( avals[apos+j] != A[aix[apos+j]][i] ) - Assert.fail("Wrong value returned by scan: "+avals[apos+j]+", expected: "+A[i][apos+aix[j]]); - } else { - if( avals[apos+j] != A[i][aix[apos+j]] ) - Assert.fail("Wrong value returned by scan: "+avals[apos+j]+", expected: "+A[i][apos+aix[j]]); - } - count++; + for(int j = 0; j < alen; j++) { + if(avals[apos + j] != A[i][aix[apos + j]]) + Assert.fail( + "Wrong value returned by scan: " + avals[apos + j] + ", expected: " + A[i][apos + aix[j]]); + count++; } } if( count != nnz ) diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockSize.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockSize.java index 0a917a9d65a..edc84098cad 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockSize.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockSize.java @@ -36,15 +36,15 @@ */ public class SparseBlockSize extends AutomatedTestBase { - private final static int rows = 324; - private final static int cols = 123; - private final static int rl = 31; - private final static int ru = 100; - private final static int cl = 30; - private final static int cu = 80; - private final static double sparsity1 = 0.12; - private final static double sparsity2 = 0.22; - private final static double sparsity3 = 0.32; + private final static int _rows = 324; + private final static int _cols = 123; + private final static int _rl = 31; + private final static int _ru = 100; + private final static int _cl = 30; + private final static int _cu = 80; + private final static double _sparsity1 = 0.12; + private final static double _sparsity2 = 0.22; + private final static double _sparsity3 = 0.32; @Override public void setUp() { @@ -53,154 +53,190 @@ public void setUp() { @Test public void testSparseBlockMCSR1() { - runSparseBlockSizeTest(SparseBlock.Type.MCSR, sparsity1); + runSparseBlockSizeTest(SparseBlock.Type.MCSR, _sparsity1); } @Test public void testSparseBlockMCSR2() { - runSparseBlockSizeTest(SparseBlock.Type.MCSR, sparsity2); + runSparseBlockSizeTest(SparseBlock.Type.MCSR, _sparsity2); } @Test public void testSparseBlockMCSR3() { - runSparseBlockSizeTest(SparseBlock.Type.MCSR, sparsity3); + runSparseBlockSizeTest(SparseBlock.Type.MCSR, _sparsity3); } @Test public void testSparseBlockCSR1() { - runSparseBlockSizeTest(SparseBlock.Type.CSR, sparsity1); + runSparseBlockSizeTest(SparseBlock.Type.CSR, _sparsity1); } @Test public void testSparseBlockCSR2() { - runSparseBlockSizeTest(SparseBlock.Type.CSR, sparsity2); + runSparseBlockSizeTest(SparseBlock.Type.CSR, _sparsity2); } @Test public void testSparseBlockCSR3() { - runSparseBlockSizeTest(SparseBlock.Type.CSR, sparsity3); + runSparseBlockSizeTest(SparseBlock.Type.CSR, _sparsity3); } @Test public void testSparseBlockCOO1() { - runSparseBlockSizeTest(SparseBlock.Type.COO, sparsity1); + runSparseBlockSizeTest(SparseBlock.Type.COO, _sparsity1); } @Test public void testSparseBlockCOO2() { - runSparseBlockSizeTest(SparseBlock.Type.COO, sparsity2); + runSparseBlockSizeTest(SparseBlock.Type.COO, _sparsity2); } @Test public void testSparseBlockCOO3() { - runSparseBlockSizeTest(SparseBlock.Type.COO, sparsity3); + runSparseBlockSizeTest(SparseBlock.Type.COO, _sparsity3); } @Test public void testSparseBlockDCSR1() { - runSparseBlockSizeTest(SparseBlock.Type.DCSR, sparsity1); + runSparseBlockSizeTest(SparseBlock.Type.DCSR, _sparsity1); } @Test public void testSparseBlockDCSR2() { - runSparseBlockSizeTest(SparseBlock.Type.DCSR, sparsity2); + runSparseBlockSizeTest(SparseBlock.Type.DCSR, _sparsity2); } @Test public void testSparseBlockDCSR3() { - runSparseBlockSizeTest(SparseBlock.Type.DCSR, sparsity3); + runSparseBlockSizeTest(SparseBlock.Type.DCSR, _sparsity3); } @Test public void testSparseBlockMCSC1(){ - runSparseBlockSizeTest(SparseBlock.Type.MCSC, sparsity1); + runSparseBlockSizeTest(SparseBlock.Type.MCSC, _sparsity1); } @Test public void testSparseBlockMCSC2(){ - runSparseBlockSizeTest(SparseBlock.Type.MCSC, sparsity2); + runSparseBlockSizeTest(SparseBlock.Type.MCSC, _sparsity2); } @Test public void testSparseBlockMCSC3(){ - runSparseBlockSizeTest(SparseBlock.Type.MCSC, sparsity3); + runSparseBlockSizeTest(SparseBlock.Type.MCSC, _sparsity3); + } + + @Test + public void testSparseBlockMCSRFixed1(){ + double[][] A = getFixedData1(); + runSparseBlockSizeTest(A, 0, 4, 0, 6, SparseBlock.Type.MCSR); + } + + @Test + public void testSparseBlockCSRFixed1(){ + double[][] A = getFixedData1(); + runSparseBlockSizeTest(A, 0, 4, 0, 6, SparseBlock.Type.CSR); + } + + @Test + public void testSparseBlockCOOFixed1(){ + double[][] A = getFixedData1(); + runSparseBlockSizeTest(A, 0, 4, 0, 6, SparseBlock.Type.COO); + } + + @Test + public void testSparseBlockMCSRFixed2(){ + double[][] A = getFixedData2(); + runSparseBlockSizeTest(A, 0, 4, 2, 4, SparseBlock.Type.MCSR); + } + + @Test + public void testSparseBlockCSRFixed2(){ + double[][] A = getFixedData2(); + runSparseBlockSizeTest(A, 0, 4, 2, 4, SparseBlock.Type.CSR); + } + + @Test + public void testSparseBlockCOOFixed2(){ + double[][] A = getFixedData2(); + runSparseBlockSizeTest(A, 0, 4, 2, 4, SparseBlock.Type.COO); + } + + @Test + public void testSparseBlockMCSRFixed3(){ + double[][] A = getFixedData3(); + runSparseBlockSizeTest(A, 0, 4, 3, 3, SparseBlock.Type.MCSR); + } + + @Test + public void testSparseBlockCSRFixed3(){ + double[][] A = getFixedData3(); + runSparseBlockSizeTest(A, 0, 4, 3, 3, SparseBlock.Type.CSR); + } + + @Test + public void testSparseBlockCOOFixed3(){ + double[][] A = getFixedData3(); + runSparseBlockSizeTest(A, 0, 4, 3, 3, SparseBlock.Type.COO); + } + + private void runSparseBlockSizeTest(SparseBlock.Type btype, double sparsity) { + double[][] A = getRandomMatrix(_rows, _cols, -10, 10, sparsity, 123); + runSparseBlockSizeTest(A, _rl, _ru, _cl, _cu, btype); } - private void runSparseBlockSizeTest( SparseBlock.Type btype, double sparsity) + private void runSparseBlockSizeTest(double[][] A, + int rl, int ru, int cl, int cu, SparseBlock.Type btype) { try { - //data generation - double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 123); - //init sparse block SparseBlock sblock = null; MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A); + int rows= mbtmp.getNumRows(); + int cols= mbtmp.getNumColumns(); + if( !mbtmp.isInSparseFormat() ) + mbtmp.denseToSparse(true); SparseBlock srtmp = mbtmp.getSparseBlock(); switch( btype ) { case MCSR: sblock = new SparseBlockMCSR(srtmp); break; case CSR: sblock = new SparseBlockCSR(srtmp); break; case COO: sblock = new SparseBlockCOO(srtmp); break; case DCSR: sblock = new SparseBlockDCSR(srtmp); break; - case MCSC: sblock = new SparseBlockMCSC(srtmp); break; + case MCSC: sblock = new SparseBlockMCSC(srtmp, cols); break; } //prepare summary statistics nnz int[] rnnz = new int[rows]; - int[] cnnz = new int[cols]; int nnz = 0; int nnz2 = 0; for( int i=0; i<rows; i++ ) { for( int j=0; j<cols; j++ ) { - cnnz[j] += (A[i][j]!=0) ? 1 : 0; rnnz[i] += (A[i][j]!=0) ? 1 : 0; - nnz2 += (i>=rl && j>=cl && i<ru && j<cu && A[i][j]!=0) ? 1 : 0; + nnz2 += (i >= rl && j >= cl && i < ru && j < cu && A[i][j] != 0) ? 1 : 0; } nnz += rnnz[i]; } - + //check full block nnz - if( nnz != sblock.size() ) - Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz); + if(nnz != sblock.size()) + Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz); //check row nnz - //for MCSC we check columns - if(sblock instanceof SparseBlockMCSC) { - for(int i = 0; i < cols; i++) - if(sblock.size(i) != cnnz[i]) { - Assert.fail("Wrong number of column non-zeros (" + i + "): " + sblock.size(i) + ", expected: " + - cnnz[i]); - } - } - else { - for(int i = 0; i < rows; i++) - if(sblock.size(i) != rnnz[i]) { - Assert.fail( - "Wrong number of row non-zeros (" + i + "): " + sblock.size(i) + ", expected: " + rnnz[i]); - } - } + for(int i = 0; i < rows; i++) + if(sblock.size(i) != rnnz[i]) + Assert.fail("Wrong number of row non-zeros (" + i + "): " + sblock.size(i) + ", expected: " + rnnz[i]); + + //check two row nnz + for(int i = 1; i < rows; i++) + if(sblock.size(i - 1, i + 1) != rnnz[i - 1] + rnnz[i]) { + Assert.fail("Wrong number of row block non-zeros (" + (i - 1) + "," + (i + 1) + "): " + + sblock.size(i - 1, i + 1) + ", expected: " + rnnz[i - 1] + rnnz[i]); + } - //check for two column nnz - if(sblock instanceof SparseBlockMCSC) { - for(int i = 1; i < cols; i++) - if(sblock.size(i - 1, i + 1) != cnnz[i - 1] + cnnz[i]) { - Assert.fail("Wrong number of column block non-zeros (" + (i - 1) + "," + (i + 1) + "): " + - sblock.size(i - 1, i + 1) + ", expected: " + cnnz[i - 1] + cnnz[i]); - } - } - else { - //check two row nnz - for(int i = 1; i < rows; i++) - if(sblock.size(i - 1, i + 1) != rnnz[i - 1] + rnnz[i]) { - Assert.fail("Wrong number of row block non-zeros (" + (i - 1) + "," + (i + 1) + "): " + - sblock.size(i - 1, i + 1) + ", expected: " + rnnz[i - 1] + rnnz[i]); - } - } - //check index range nnz - if( sblock.size(rl, ru, cl, cu) != nnz2 ) - Assert.fail("Wrong number of range non-zeros: " + - sblock.size(rl, ru, cl, cu)+", expected: "+nnz2); + if(sblock.size(rl, ru, cl, cu) != nnz2) + Assert.fail("Wrong number of range non-zeros: " + sblock.size(rl, ru, cl, cu) + ", expected: " + nnz2); } catch(Exception ex) { @@ -208,4 +244,28 @@ private void runSparseBlockSizeTest( SparseBlock.Type btype, double sparsity) throw new RuntimeException(ex); } } + + private double[][] getFixedData1(){ + return new double[][] + {{10, 20, 0, 0, 0, 0}, + {0, 30, 0, 40, 0, 0}, + {0, 0, 50, 60, 70, 0}, + {0, 0, 0, 0, 0, 80}}; + } + + private double[][] getFixedData2(){ + return new double[][] + {{10, 10, 0, 0, 0, 20}, + {10, 10, 0, 0, 0, 20}, + {10, 10, 0, 0, 0, 20}, + {10, 10, 0, 0, 0, 20}}; + } + + private double[][] getFixedData3(){ + return new double[][] + {{10, 10, 0, 15, 0, 20}, + {10, 10, 0, 15, 0, 20}, + {10, 10, 0, 15, 0, 20}, + {10, 10, 0, 15, 0, 20}}; + } }