diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java index 1ad2cd57d01..d83a9263dc2 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java @@ -312,15 +312,15 @@ public int size(int r) { public long size(int rl, int ru) { return pos(ru) - pos(rl); } - + @Override public long size(int rl, int ru, int cl, int cu) { long nnz = 0; - for(int i=rl; i SparseBlock.INIT_CAPACITY && _columns[c].size() * SparseBlock.RESIZE_FACTOR1 < ((SparseRowVector) _columns[c]).capacity()) { @@ -272,13 +298,11 @@ else if(_columns[c] instanceof SparseRowScalar) { _columns[c] = null; } } - } @Override public int numRows() { - // this is a column-oriented layout - return 0; + return _rlen; } public int numCols() { @@ -296,7 +320,14 @@ public boolean isContiguous() { } @Override - public boolean isAllocated(int c) { + public boolean isAllocated(int r) { + for(SparseRow col : _columns) + if(col == null) + return false; + return true; + } + + public boolean isAllocatedCol(int c) { return _columns[c] != null; } @@ -319,8 +350,19 @@ public void reset(int ennz, int maxnnz) { } @Override - public void reset(int c, int ennz, int maxnnz) { - if(isAllocated(c)) { + public void reset(int r, int ennz, int maxnnz) { + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i)) { + if(_columns[i] instanceof SparseRowScalar && _columns[i].indexes()[0] == r) + _columns[i].set(r, 0); + else if(_columns[i] instanceof SparseRowVector) + _columns[i].set(r, 0); + } + } + } + + public void resetCol(int c, int ennz, int maxnnz) { + if(isAllocatedCol(c)) { _columns[c].reset(ennz, maxnnz); } } @@ -337,13 +379,33 @@ public long size() { } @Override - public int size(int c) { + public int size(int r) { + int nnz = 0; + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i)) + nnz += (_columns[i].get(r) != 0) ? 1 : 0; + } + return nnz; + } + + public int sizeCol(int c) { //prior check with isEmpty(r) expected return isAllocated(c) ? _columns[c].size() : 0; } @Override - public long size(int cl, int cu) { + public long size(int rl, int ru) { + long nnz = 0; + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i)) { + for(int j = rl; j < ru; j++) + nnz += (_columns[i].get(j) != 0) ? 1 : 0; + } + } + return nnz; + } + + public long sizeCol(int cl, int cu) { long nnz = 0; for(int i = cl; i < cu; i++) { nnz += isAllocated(i) ? _columns[i].size() : 0; @@ -355,17 +417,27 @@ public long size(int cl, int cu) { public long size(int rl, int ru, int cl, int cu) { long nnz = 0; for(int i = cl; i < cu; i++) { - if(!isEmpty(i)) { - int start = posFIndexGTE(rl, i); - int end = posFIndexGTE(ru, i); - nnz += (start != -1) ? (end - start) : 0; + if(!isEmptyCol(i)) { + int start = posFIndexGTECol(rl, i); + int end = posFIndexLTECol(ru - 1, i); + nnz += (start != -1 && end != -1) ? (end - start + 1) : 0; } } return nnz; } @Override - public boolean isEmpty(int c) { + public boolean isEmpty(int r) { + for(int i = 0; i < _columns.length; i++) { + if(!isAllocatedCol(i)) + continue; + else if(_columns[i].get(r) != 0) + return false; + } + return true; + } + + public boolean isEmptyCol(int c) { return _columns[c] == null || _columns[c].isEmpty(); } @@ -413,26 +485,54 @@ public boolean checkValidity(int rlen, int clen, long nnz, boolean strict) { } @Override - public int[] indexes(int c) { + public int[] indexes(int r) { + //prior check with isEmpty(r) expected + int nnz = size(r); + int[] idx = new int[nnz]; + int index = 0; + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i) && _columns[i].get(r) != 0) { + idx[index] = i; + index++; + } + } + return idx; + } + + public int[] indexesCol(int c) { //prior check with isEmpty(c) expected return _columns[c].indexes(); } @Override - public double[] values(int c) { + public double[] values(int r) { + //prior check with isEmpty(r) expected + int nnz = size(r); + double[] vals = new double[nnz]; + int index = 0; + for(int i = 0; i < _columns.length; i++) { + if(isAllocatedCol(i) && _columns[i].get(r) != 0) { + vals[index] = _columns[i].get(r); + index++; + } + } + return vals; + } + + public double[] valuesCol(int c) { //prior check with isEmpty(c) expected return _columns[c].values(); } @Override - public int pos(int c) { - //arrays per column (always start 0) + public int pos(int r) { + //arrays per row (always start 0) return 0; } @Override public boolean set(int r, int c, double v) { - if(!isAllocated(c)) { + if(!isAllocatedCol(c)) { _columns[c] = new SparseRowScalar(); } else if(_columns[c] instanceof SparseRowScalar && !_columns[c].isEmpty()) { @@ -442,9 +542,17 @@ else if(_columns[c] instanceof SparseRowScalar && !_columns[c].isEmpty()) { } @Override - public void set(int c, SparseRow col, boolean deep) { + public void set(int r, SparseRow row, boolean deep) { + reset(r, 1, 1); + int nnz = row.size(); + for(int i = 0; i < nnz; i++) { + set(r, row.indexes()[i], row.values()[i]); + } + } + + public void setCol(int c, SparseRow col, boolean deep) { //copy values into existing column to avoid allocation - if(isAllocated(c) && _columns[c] instanceof SparseRowVector && + if(isAllocatedCol(c) && _columns[c] instanceof SparseRowVector && ((SparseRowVector) _columns[c]).capacity() >= col.size() && deep) { ((SparseRowVector) _columns[c]).copy(col); //set new sparse column (incl allocation if required) @@ -456,7 +564,7 @@ public void set(int c, SparseRow col, boolean deep) { @Override public boolean add(int r, int c, double v) { - if(!isAllocated(c)) { + if(!isAllocatedCol(c)) { _columns[c] = new SparseRowScalar(); } else if(_columns[c] instanceof SparseRowScalar && !_columns[c].isEmpty()) { @@ -485,8 +593,16 @@ else if(_columns[c] == null) { } @Override - public void setIndexRange(int c, int rl, int ru, double[] v, int vix, int vlen) { - if(!isAllocated(c)) { + public void setIndexRange(int r, int cl, int cu, double[] v, int vix, int vlen) { + int idx = vix; + for(int i = cl; i < cu; i++) { + set(r, i, v[idx]); + idx++; + } + } + + public void setIndexRangeCol(int c, int rl, int ru, double[] v, int vix, int vlen) { + if(!isAllocatedCol(c)) { _columns[c] = new SparseRowVector(); } else if(_columns[c] instanceof SparseRowScalar) { @@ -496,8 +612,14 @@ else if(_columns[c] instanceof SparseRowScalar) { } @Override - public void setIndexRange(int c, int rl, int ru, double[] v, int[] vix, int vpos, int vlen) { - if(!isAllocated(c)) { + public void setIndexRange(int r, int cl, int cu, double[] v, int[] vix, int vpos, int vlen) { + for(int i = vpos; i < (vpos + vlen); i++) { + set(r, vix[i], v[i]); + } + } + + public void setIndexRangeCol(int c, int rl, int ru, double[] v, int[] vix, int vpos, int vlen) { + if(!isAllocatedCol(c)) { _columns[c] = new SparseRowVector(); } else if(_columns[c] instanceof SparseRowScalar) { @@ -508,7 +630,18 @@ else if(_columns[c] instanceof SparseRowScalar) { } @Override - public void deleteIndexRange(int c, int rl, int ru) { + public void deleteIndexRange(int r, int cl, int cu) { + for(int i = cl; i < cu; i++) { + if(isAllocatedCol(i)) { + if(_columns[i] instanceof SparseRowScalar && _columns[i].indexes()[0] == r) + _columns[i].set(r, 0); + else if(_columns[i] instanceof SparseRowVector) + _columns[i].set(r, 0); + } + } + } + + public void deleteIndexRangeCol(int c, int rl, int ru) { //prior check with isEmpty(c) expected //different sparse row semantics: upper bound inclusive if(_columns[c] instanceof SparseRowScalar) { @@ -527,26 +660,48 @@ public void sort() { } @Override - public void sort(int c) { + public void sort(int r) { + //prior check with isEmpty(c) expected + sort(); + } + + public void sortCol(int c) { //prior check with isEmpty(c) expected _columns[c].sort(); } @Override public double get(int r, int c) { - if(!isAllocated(c)) { + if(!isAllocatedCol(c)) { return 0; } return _columns[c].get(r); } @Override - public SparseRow get(int c) { + public SparseRow get(int r) { + SparseRow row = (size(r) == 1) ? new SparseRowScalar() : new SparseRowVector(size(r)); + double v = 0; + for(int i = 0; i < _columns.length; i++) { + v = get(r, i); + if(v != 0) + row.set(i, v); + } + return row; + } + + public SparseRow getCol(int c) { return _columns[c]; } @Override public int posFIndexLTE(int r, int c) { + //prior check with isEmpty(r) expected + SparseRow row = get(r); + return ((SparseRowVector) row).searchIndexesFirstLTE(c); + } + + public int posFIndexLTECol(int r, int c) { //prior check with isEmpty(c) expected if(_columns[c] instanceof SparseRowScalar) { _columns[c] = new SparseRowVector(_columns[c]); @@ -556,17 +711,49 @@ public int posFIndexLTE(int r, int c) { @Override public int posFIndexGTE(int r, int c) { + SparseRow row = get(r); + return row.searchIndexesFirstGTE(c); + } + + public int posFIndexGTECol(int r, int c) { return _columns[c].searchIndexesFirstGTE(r); } @Override public int posFIndexGT(int r, int c) { + SparseRow row = get(r); + return row.searchIndexesFirstGT(c); + } + + public int posFIndexGTCol(int r, int c) { return _columns[c].searchIndexesFirstGT(r); } @Override public Iterator getNonEmptyRowsIterator(int rl, int ru) { - throw new UnsupportedOperationException("Non-empty rows iterator is not supported in column layouts."); + return new NonEmptyRowsIteratorMCSC(rl, ru); + } + + public class NonEmptyRowsIteratorMCSC implements Iterator { + private int _rpos; + private final int _ru; + + public NonEmptyRowsIteratorMCSC(int rl, int ru) { + _rpos = rl; + _ru = ru; + } + + @Override + public boolean hasNext() { + while(_rpos < _ru && isEmpty(_rpos)) + _rpos++; + return _rpos < _ru; + } + + @Override + public Integer next() { + return _rpos++; + } } @Override @@ -580,7 +767,7 @@ public String toString() { sb.append("\n"); final int colDigits = (int) Math.max(Math.ceil(Math.log10(nCol)), 1); for(int i = 0; i < nCol; i++) { - if(isEmpty(i)) + if(isEmptyCol(i)) continue; sb.append(String.format("%0" + colDigits + "d %s\n", i, _columns[i].toString())); } @@ -588,7 +775,71 @@ public String toString() { return sb.toString(); } + /** + * Helper function for MCSC + * + * @return the underlying array of columns {@link SparseRow} + */ public SparseRow[] getCols() { return _columns; } + + /** + * Helper function for MCSC + * + * @return the corresponding array of rows {@link SparseRow} + */ + + public SparseRow[] getRows() { + SparseRow[] rows = new SparseRow[numRows()]; + for(int i = 0; i < numRows(); i++) { + rows[i] = get(i); + } + return rows; + } + + public Iterator getNonEmptyColumnsIterator(int cl, int cu) { + return new NonEmptyColumnsIteratorMCSC(cl, cu); + } + + public class NonEmptyColumnsIteratorMCSC implements Iterator { + private int _cpos; + private final int _cu; + + public NonEmptyColumnsIteratorMCSC(int cl, int cu) { + _cpos = cl; + _cu = cu; + } + + @Override + public boolean hasNext() { + while(_cpos < _cu && isEmptyCol(_cpos)) { + _cpos++; + } + return _cpos < _cu; + } + + @Override + public Integer next() { + return _cpos++; + } + + } + + @SuppressWarnings("unused") + private class SparseNonEmptyColumnIterable implements Iterable { + private final int _cl; //column lower + private final int _cu; //column upper + + protected SparseNonEmptyColumnIterable(int cl, int cu) { + _cl = cl; + _cu = cu; + } + + @Override + public Iterator iterator() { + //use specialized non-empty row iterators of sparse blocks + return getNonEmptyColumnsIterator(_cl, _cu); + } + } } diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java index 52b5d2e338a..d232a0041a6 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java @@ -305,15 +305,15 @@ public long size(int rl, int ru) { ret += isAllocated(i) ? _rows[i].size() : 0; return ret; } - + @Override public long size(int rl, int ru, int cl, int cu) { long nnz = 0; - for(int i=rl; i iter = map.getIterator(); - while( iter.hasNext() ) { //random hash order + while(iter.hasNext()) { //random hash order ADoubleEntry e = iter.next(); - sblock.append((int)e.getKey1(), (int)e.getKey2(), e.value); + sblock.append((int) e.getKey1(), (int) e.getKey2(), e.value); } - } - + } + //sort appended values sblock.sort(); - + //check for correct number of non-zeros - int[] rnnz = new int[rows]; - int nnz = 0; - int[] cnnz = new int[cols]; - for( int i=0; i= 0) ? - sblock.indexes(i)[sblock.pos(i) + sixpos] : -1; - if (six != ix) { - Assert.fail("Wrong index returned by index probe (" + - itype.toString() + "," + i + "): " + six + ", expected: " + ix); - } + for(int i = 0; i < rows; i++) { + int ix = getFirstIx(A, i, i, itype); + int sixpos = -1; + switch(itype) { + case GT: + sixpos = sblock.posFIndexGT(i, i); + break; + case GTE: + sixpos = sblock.posFIndexGTE(i, i); + break; + case LTE: + sixpos = sblock.posFIndexLTE(i, i); + break; } - } - else{ - for( int i=0; i=0) ? - sblock.indexes(i)[sblock.pos(i)+sixpos] : -1; - if( six != ix ) { - Assert.fail("Wrong index returned by index probe ("+ - itype.toString()+","+i+"): "+six+", expected: "+ix); - } + int six = (sixpos >= 0) ? sblock.indexes(i)[sblock.pos(i) + sixpos] : -1; + if(six != ix) { + Assert.fail("Wrong index returned by index probe (" + itype.toString() + "," + i + "): " + + six + ", expected: " + ix); } } + } catch(Exception ex) { ex.printStackTrace(); @@ -391,27 +380,4 @@ else if( type==IndexType.LTE ) { return -1; } - private static int getFirstIxCol(double[][] A, int rix, int cix, IndexType type) { - if(type == IndexType.GT) { - for(int j = rix + 1; j < rows; j++) - if(A[j][cix] != 0) - return j; - return -1; - } - else if(type == IndexType.GTE) { - for(int j = rix; j < rows; j++) - if(A[j][cix] != 0) - return j; - return -1; - } - else if(type == IndexType.LTE) { - for(int j = rix; j >= 0; j--) - if(A[j][cix] != 0) - return j; - return -1; - } - - return -1; - } - } diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetSet.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetSet.java index bb284d34a47..2f357d3c8ab 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetSet.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockGetSet.java @@ -244,47 +244,47 @@ public void testSparseBlockDCSR3Rand() { @Test public void testSparseBlockMCSC1Bulk() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity1, InitType.BULK); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity1, InitType.BULK); } @Test public void testSparseBlockMCSC2Bulk() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity2, InitType.BULK); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity2, InitType.BULK); } @Test public void testSparseBlockMCSC3Bulk() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity3, InitType.BULK); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity3, InitType.BULK); } @Test public void testSparseBlockMCSC1Seq() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity1, InitType.SEQ_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity1, InitType.SEQ_SET); } @Test public void testSparseBlockMCSC2Seq() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity2, InitType.SEQ_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity2, InitType.SEQ_SET); } @Test public void testSparseBlockMCSC3Seq() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity3, InitType.SEQ_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity3, InitType.SEQ_SET); } @Test public void testSparseBlockMCSC1Rand() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity1, InitType.RAND_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity1, InitType.RAND_SET); } @Test public void testSparseBlockMCSC2Rand() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity2, InitType.RAND_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity2, InitType.RAND_SET); } @Test public void testSparseBlockMCSC3Rand() { - runSparseBlockGetSetColumnTest(SparseBlock.Type.MCSC, sparsity3, InitType.RAND_SET); + runSparseBlockGetSetTest(SparseBlock.Type.MCSC, sparsity3, InitType.RAND_SET); } private void runSparseBlockGetSetTest( SparseBlock.Type btype, double sparsity, InitType itype) @@ -310,10 +310,6 @@ else if( itype == InitType.SEQ_SET || itype == InitType.RAND_SET ) { case MCSC: sblock = new SparseBlockMCSC(rows, cols); break; } - if(sblock instanceof SparseBlockMCSC){ - - } - if(itype == InitType.SEQ_SET) { for( int i=0; i iter = map.getIterator(); - while(iter.hasNext()) { //random hash order - ADoubleEntry e = iter.next(); - int r = (int) e.getKey1(); - int c = (int) e.getKey2(); - sblock.set(r, c, e.value); - } - } - } - - //check basic meta data - if(sblock.numCols() != cols) - Assert.fail("Wrong number of cols: " + sblock.numCols() + ", expected: " + cols); - - //check for correct number of non-zeros - int[] cnnz = new int[cols]; - int nnz = 0; - for(int i = 0; i < cols; i++) { - for(int j = 0; j < rows; j++) - cnnz[i] += (A[j][i] != 0) ? 1 : 0; - nnz += cnnz[i]; - } - if(nnz != sblock.size()) - Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz); - - //check correct isEmpty return - for(int i = 0; i < cols; i++) - if(sblock.isEmpty(i) != (cnnz[i] == 0)) - Assert.fail("Wrong isEmpty(col) result for row nnz: " + cnnz[i] + "(column: " + i + ")"); - - //check correct values - for(int i = 0; i < cols; i++) - if(!sblock.isEmpty(i)) - for(int j = 0; j < rows; j++) { - double tmp = sblock.get(j, i); - if(tmp != A[j][i]) - Assert.fail( - "Wrong get value for cell (" + i + "," + j + "): " + tmp + ", expected: " + A[j][i]); - } - - } - catch(Exception ex) { - ex.printStackTrace(); - throw new RuntimeException(ex); - } - } - } diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIndexRange.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIndexRange.java index 4a87f0d129e..d51f9d1f6da 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIndexRange.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIndexRange.java @@ -43,8 +43,6 @@ public class SparseBlockIndexRange extends AutomatedTestBase private final static int cols = 549; private final static int cl = 245; private final static int cu = 425; - private final static int rl = 245; - private final static int ru = 525; private final static double sparsity1 = 0.12; private final static double sparsity2 = 0.22; private final static double sparsity3 = 0.32; @@ -181,32 +179,32 @@ public void testSparseBlockDCSR3Insert() { @Test public void testSparseBlockMCSC1Delete() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity1, UpdateType.DELETE); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity1, UpdateType.DELETE); } @Test public void testSparseBlockMCSC2Delete() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity2, UpdateType.DELETE); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity2, UpdateType.DELETE); } @Test public void testSparseBlockMCSC3Delete() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity3, UpdateType.DELETE); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity3, UpdateType.DELETE); } @Test public void testSparseBlockMCSC1Insert() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity1, UpdateType.INSERT); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity1, UpdateType.INSERT); } @Test public void testSparseBlockMCSC2Insert() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity2, UpdateType.INSERT); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity2, UpdateType.INSERT); } @Test public void testSparseBlockMCSC3Insert() { - runSparseBlockIndexRangeColumnTest(SparseBlock.Type.MCSC, sparsity3, UpdateType.INSERT); + runSparseBlockIndexRangeTest(SparseBlock.Type.MCSC, sparsity3, UpdateType.INSERT); } private void runSparseBlockIndexRangeTest( SparseBlock.Type btype, double sparsity, UpdateType utype) @@ -278,64 +276,4 @@ else if( utype == UpdateType.INSERT ) { } } - @SuppressWarnings("incomplete-switch") - private void runSparseBlockIndexRangeColumnTest(SparseBlock.Type btype, double sparsity, UpdateType utype) { - try { - //data generation - double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 456); - - //init sparse block - SparseBlock sblock = null; - MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A); - SparseBlock srtmp = mbtmp.getSparseBlock(); - switch(btype) { - case MCSC: - sblock = new SparseBlockMCSC(srtmp, cols); - break; - } - - //delete range per row via set - if(utype == UpdateType.DELETE) { - for(int i = 0; i < cols; i++) { - sblock.deleteIndexRange(i, rl, ru); - for(int j = rl; j < ru; j++) { - A[j][i] = 0; // Fill column-wise with zeros - } - } - } - else if(utype == UpdateType.INSERT) { - double[] vals = new double[ru - rl]; - for(int j = rl; j < ru; j++) - vals[j - rl] = j; - for(int i = 0; i < cols; i++) { - sblock.setIndexRange(i, rl, ru, vals, 0, ru - rl); - for(int j = rl; j < ru; j++) { - A[j][i] = vals[j - rl]; // Update the matrix column-wise - } - } - } - - //check for correct number of non-zeros - int[] cnnz = new int[cols]; - int nnz = 0; - for(int i = 0; i < cols; i++) { - for(int j = 0; j < rows; j++) - cnnz[i] += (A[j][i] != 0) ? 1 : 0; - nnz += cnnz[i]; - } - if(nnz != sblock.size()) - Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz); - - //check correct isEmpty return - for(int i = 0; i < cols; i++) - if(sblock.isEmpty(i) != (cnnz[i] == 0)) - Assert.fail("Wrong isEmpty(col) result for row nnz: " + cnnz[i]); - - //TODO: Add 'check correct values', requires Iterator - } - catch(Exception ex) { - ex.printStackTrace(); - throw new RuntimeException(ex); - } - } } diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIterator.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIterator.java index 0039d5ac1ea..736f981025d 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIterator.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockIterator.java @@ -172,6 +172,36 @@ public void testSparseBlockDCSR3Partial() { runSparseBlockIteratorTest(SparseBlock.Type.DCSR, sparsity3, true); } + @Test + public void testSparseBlockMCSC1Full() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity1, false); + } + + @Test + public void testSparseBlockMCSC2Full() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity2, false); + } + + @Test + public void testSparseBlockMCSC3Full() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity3, false); + } + + @Test + public void testSparseBlockMCSC1Partial() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity1, true); + } + + @Test + public void testSparseBlockMCSC2Partial() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity2, true); + } + + @Test + public void testSparseBlockMCSC3Partial() { + runSparseBlockIteratorTest(SparseBlock.Type.MCSC, sparsity3, true); + } + private void runSparseBlockIteratorTest(SparseBlock.Type btype, double sparsity, boolean partial) { try { //data generation @@ -181,7 +211,7 @@ private void runSparseBlockIteratorTest(SparseBlock.Type btype, double sparsity, MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A); SparseBlock srtmp = mbtmp.getSparseBlock(); SparseBlock sblock = SparseBlockFactory.copySparseBlock(btype, srtmp, true); - + //check for correct number of non-zeros int[] rnnz = new int[rows]; int nnz = 0; @@ -215,9 +245,9 @@ private void runSparseBlockIteratorTest(SparseBlock.Type btype, double sparsity, // check iterator over non-zero rows List manualNonZeroRows = new ArrayList<>(); List iteratorNonZeroRows = new ArrayList<>(); - Iterator iterRows = !partial ? - sblock.getNonEmptyRowsIterator(0, rows) : - sblock.getNonEmptyRowsIterator(rl, rows); + Iterator iterRows = !partial + ? sblock.getNonEmptyRowsIterator(0, rows) + : sblock.getNonEmptyRowsIterator(rl, rows); for(int i = rl; i < rows; i++) if(!sblock.isEmpty(i)) @@ -230,6 +260,7 @@ private void runSparseBlockIteratorTest(SparseBlock.Type btype, double sparsity, if(!manualNonZeroRows.equals(iteratorNonZeroRows)) { Assert.fail("Verification of iterator over non-zero rows failed."); } + } catch(Exception ex) { ex.printStackTrace(); diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMemEstimate.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMemEstimate.java index 36dc077fb06..1c0c2aa7c5c 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMemEstimate.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMemEstimate.java @@ -98,7 +98,7 @@ private static void runSparseBlockMemoryTest( double sparsity) } else { //ultra-sparse (pref COO) if( memMCSC < memCOO ) - Assert.fail("SparseBlockMCS memory estimate smaller than SparseBlockCOO estimate."); + Assert.fail("SparseBlockMCSC memory estimate smaller than SparseBlockCOO estimate."); if( memMCSR < memCOO ) Assert.fail("SparseBlockMCSR memory estimate smaller than SparseBlockCOO estimate."); if( memCSR < memCOO ) diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMerge.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMerge.java index e5681e294e9..7c061eaf279 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMerge.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockMerge.java @@ -122,6 +122,26 @@ public void testMergeMCSR_DCSR_2() { public void testMergeMCSR_DCSR_3() { runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.DCSR, sparsity3); } + + @Test + public void testMergeMCSR_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeMCSR_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeMCSR_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeMCSR_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSR, SparseBlock.Type.MCSC, sparsity3); + } @Test public void testMergeCSR_CSR_0() { @@ -202,6 +222,26 @@ public void testMergeCSR_COO_2() { public void testMergeCSR_COO_3() { runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.COO, sparsity3); } + + @Test + public void testMergeCSR_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeCSR_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeCSR_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeCSR_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.CSR, SparseBlock.Type.MCSC, sparsity3); + } @Test public void testMergeCOO_COO_0() { @@ -283,6 +323,26 @@ public void testMergeCOO_DCSR_3() { runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.DCSR, sparsity3); } + @Test + public void testMergeCOO_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeCOO_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeCOO_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeCOO_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.COO, SparseBlock.Type.MCSC, sparsity3); + } + @Test public void testMergeDCSR_DCSR_0() { runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.DCSR, sparsity0); @@ -363,7 +423,108 @@ public void testMergeDCSR_COO_3() { runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.COO, sparsity3); } - + @Test + public void testMergeDCSR_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeDCSR_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeDCSR_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeDCSR_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.DCSR, SparseBlock.Type.MCSC, sparsity3); + } + + @Test + public void testMergeMCSC_MCSC_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSC, sparsity0); + } + + @Test + public void testMergeMCSC_MCSC_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSC, sparsity1); + } + + @Test + public void testMergeMCSC_MCSC_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSC, sparsity2); + } + + @Test + public void testMergeMCSC_MCSC_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSC, sparsity3); + } + + @Test + public void testMergeMCSC_CSR_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.CSR, sparsity0); + } + + @Test + public void testMergeMCSC_CSR_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.CSR, sparsity1); + } + + @Test + public void testMergeMCSC_CSR_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.CSR, sparsity2); + } + + @Test + public void testMergeMCSC_CSR_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.CSR, sparsity3); + } + + @Test + public void testMergeMCSC_MCSR_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSR, sparsity0); + } + + @Test + public void testMergeMCSC_MCSR_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSR, sparsity1); + } + + @Test + public void testMergeMCSC_MCSR_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSR, sparsity2); + } + + @Test + public void testMergeMCSC_MCSR_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.MCSR, sparsity3); + } + + @Test + public void testMergeMCSC_COO_0() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.COO, sparsity0); + } + + @Test + public void testMergeMCSC_COO_1() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.COO, sparsity1); + } + + @Test + public void testMergeMCSC_COO_2() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.COO, sparsity2); + } + + @Test + public void testMergeMCSC_COO_3() { + runSparseBlockMergeTest(SparseBlock.Type.MCSC, SparseBlock.Type.COO, sparsity3); + } + + + private void runSparseBlockMergeTest( SparseBlock.Type btype1, SparseBlock.Type btype2, double sparsity) { try @@ -387,8 +548,8 @@ private void runSparseBlockMergeTest( SparseBlock.Type btype1, SparseBlock.Type MatrixBlock mb1 = DataConverter.convertToMatrixBlock(B1); MatrixBlock mb2 = DataConverter.convertToMatrixBlock(B2); long nnz = mb1.getNonZeros() + mb2.getNonZeros(); - mb1.setSparseBlock(SparseBlockFactory.copySparseBlock(btype1, mb1.getSparseBlock(), false)); - mb2.setSparseBlock(SparseBlockFactory.copySparseBlock(btype2, mb2.getSparseBlock(), false)); + mb1.setSparseBlock(SparseBlockFactory.copySparseBlock(btype1, mb1.getSparseBlock(), false, cols)); + mb2.setSparseBlock(SparseBlockFactory.copySparseBlock(btype2, mb2.getSparseBlock(), false, cols)); //execute merge mb1 = mb1.merge(mb2, false); diff --git a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockScan.java b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockScan.java index 98fe32792ed..8163b49a6ef 100644 --- a/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockScan.java +++ b/src/test/java/org/apache/sysds/test/component/sparse/SparseBlockScan.java @@ -142,48 +142,32 @@ private void runSparseBlockScanTest( SparseBlock.Type btype, double sparsity) //check for correct number of non-zeros int[] rnnz = new int[rows]; int nnz = 0; - int[] cnnz = new int[cols]; - for( int i=0; i=rl && j>=cl && i= rl && j >= cl && i < ru && j < cu && A[i][j] != 0) ? 1 : 0; } nnz += rnnz[i]; } - + //check full block nnz - if( nnz != sblock.size() ) - Assert.fail("Wrong number of non-zeros: "+sblock.size()+", expected: "+nnz); + if(nnz != sblock.size()) + Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz); //check row nnz - //for MCSC we check columns - if(sblock instanceof SparseBlockMCSC) { - for(int i = 0; i < cols; i++) - if(sblock.size(i) != cnnz[i]) { - Assert.fail("Wrong number of column non-zeros (" + i + "): " + sblock.size(i) + ", expected: " + - cnnz[i]); - } - } - else { - for(int i = 0; i < rows; i++) - if(sblock.size(i) != rnnz[i]) { - Assert.fail( - "Wrong number of row non-zeros (" + i + "): " + sblock.size(i) + ", expected: " + rnnz[i]); - } - } + for(int i = 0; i < rows; i++) + if(sblock.size(i) != rnnz[i]) + Assert.fail("Wrong number of row non-zeros (" + i + "): " + sblock.size(i) + ", expected: " + rnnz[i]); + + //check two row nnz + for(int i = 1; i < rows; i++) + if(sblock.size(i - 1, i + 1) != rnnz[i - 1] + rnnz[i]) { + Assert.fail("Wrong number of row block non-zeros (" + (i - 1) + "," + (i + 1) + "): " + + sblock.size(i - 1, i + 1) + ", expected: " + rnnz[i - 1] + rnnz[i]); + } - //check for two column nnz - if(sblock instanceof SparseBlockMCSC) { - for(int i = 1; i < cols; i++) - if(sblock.size(i - 1, i + 1) != cnnz[i - 1] + cnnz[i]) { - Assert.fail("Wrong number of column block non-zeros (" + (i - 1) + "," + (i + 1) + "): " + - sblock.size(i - 1, i + 1) + ", expected: " + cnnz[i - 1] + cnnz[i]); - } - } - else { - //check two row nnz - for(int i = 1; i < rows; i++) - if(sblock.size(i - 1, i + 1) != rnnz[i - 1] + rnnz[i]) { - Assert.fail("Wrong number of row block non-zeros (" + (i - 1) + "," + (i + 1) + "): " + - sblock.size(i - 1, i + 1) + ", expected: " + rnnz[i - 1] + rnnz[i]); - } - } - //check index range nnz - if( sblock.size(rl, ru, cl, cu) != nnz2 ) - Assert.fail("Wrong number of range non-zeros: " + - sblock.size(rl, ru, cl, cu)+", expected: "+nnz2); + if(sblock.size(rl, ru, cl, cu) != nnz2) + Assert.fail("Wrong number of range non-zeros: " + sblock.size(rl, ru, cl, cu) + ", expected: " + nnz2); } catch(Exception ex) { @@ -208,4 +244,28 @@ private void runSparseBlockSizeTest( SparseBlock.Type btype, double sparsity) throw new RuntimeException(ex); } } + + private double[][] getFixedData1(){ + return new double[][] + {{10, 20, 0, 0, 0, 0}, + {0, 30, 0, 40, 0, 0}, + {0, 0, 50, 60, 70, 0}, + {0, 0, 0, 0, 0, 80}}; + } + + private double[][] getFixedData2(){ + return new double[][] + {{10, 10, 0, 0, 0, 20}, + {10, 10, 0, 0, 0, 20}, + {10, 10, 0, 0, 0, 20}, + {10, 10, 0, 0, 0, 20}}; + } + + private double[][] getFixedData3(){ + return new double[][] + {{10, 10, 0, 15, 0, 20}, + {10, 10, 0, 15, 0, 20}, + {10, 10, 0, 15, 0, 20}, + {10, 10, 0, 15, 0, 20}}; + } }