@Override public void collect(Long rowIndex, Vector v) throws IOException { long blockKey = rowIndex / height; if (blockKey != currentBlockNum) { flushBlock(); if (block == null) { block = new SparseRowBlockWritable(100); } currentBlockNum = blockKey; } block.plusRow((int) (rowIndex % height), v); }
private void flushBlock() throws IOException { if (block == null || block.getNumRows() == 0) { return; } blockKeyW.set(currentBlockNum); delegate.collect(blockKeyW, block); block.clear(); }
@Override protected void reduce(SplitPartitionedWritable key, Iterable<SparseRowBlockWritable> values, Context context) throws IOException, InterruptedException { accum.clear(); for (SparseRowBlockWritable bw : values) { accum.plusBlock(bw); } if (key.getTaskId() != lastTaskId) { setupBlock(context, key); } long blockBase = key.getTaskItemOrdinal() * blockHeight; for (int k = 0; k < accum.getNumRows(); k++) { Vector yiRow = accum.getRows()[k]; key.setTaskItemOrdinal(blockBase + accum.getRowIndices()[k]); qr.collect(key, yiRow); } }
public void plusRow(int index, Vector row) { /* * often accumulation goes in row-increasing order, so check for this to * avoid binary search (another log Height multiplier). */ int pos = numRows == 0 || rowIndices[numRows - 1] < index ? -numRows - 1 : Arrays .binarySearch(rowIndices, 0, numRows, index); if (pos >= 0) { rows[pos].assign(row, PlusMult.plusMult(1)); } else { insertIntoPos(-pos - 1, index, row); } }
@Override protected void reduce(SplitPartitionedWritable key, Iterable<SparseRowBlockWritable> values, Context context) throws IOException, InterruptedException { accum.clear(); for (SparseRowBlockWritable bw : values) { accum.plusBlock(bw); } if (key.getTaskId() != lastTaskId) { setupBlock(context, key); } long blockBase = key.getTaskItemOrdinal() * blockHeight; for (int k = 0; k < accum.getNumRows(); k++) { Vector yiRow = accum.getRows()[k]; key.setTaskItemOrdinal(blockBase + accum.getRowIndices()[k]); qr.collect(key, yiRow); } }
public void plusRow(int index, Vector row) { /* * often accumulation goes in row-increasing order, so check for this to * avoid binary search (another log Height multiplier). */ int pos = numRows == 0 || rowIndices[numRows - 1] < index ? -numRows - 1 : Arrays .binarySearch(rowIndices, 0, numRows, index); if (pos >= 0) { rows[pos].assign(row, PlusMult.plusMult(1)); } else { insertIntoPos(-pos - 1, index, row); } }
@Override protected void reduce(SplitPartitionedWritable key, Iterable<SparseRowBlockWritable> values, Context context) throws IOException, InterruptedException { accum.clear(); for (SparseRowBlockWritable bw : values) { accum.plusBlock(bw); } if (key.getTaskId() != lastTaskId) { setupBlock(context, key); } long blockBase = key.getTaskItemOrdinal() * blockHeight; for (int k = 0; k < accum.getNumRows(); k++) { Vector yiRow = accum.getRows()[k]; key.setTaskItemOrdinal(blockBase + accum.getRowIndices()[k]); qr.collect(key, yiRow); } }
@Override public void collect(Long rowIndex, Vector v) throws IOException { long blockKey = rowIndex / height; if (blockKey != currentBlockNum) { flushBlock(); if (block == null) { block = new SparseRowBlockWritable(100); } currentBlockNum = blockKey; } block.plusRow((int) (rowIndex % height), v); }
private void flushBlock() throws IOException { if (block == null || block.getNumRows() == 0) { return; } blockKeyW.set(currentBlockNum); delegate.collect(blockKeyW, block); block.clear(); }
public void plusRow(int index, Vector row) { /* * often accumulation goes in row-increasing order, so check for this to * avoid binary search (another log Height multiplier). */ int pos = numRows == 0 || rowIndices[numRows - 1] < index ? -numRows - 1 : Arrays .binarySearch(rowIndices, 0, numRows, index); if (pos >= 0) { rows[pos].assign(row, PlusMult.plusMult(1)); } else { insertIntoPos(-pos - 1, index, row); } }
InterruptedException { accum.clear(); for (SparseRowBlockWritable bw : values) { accum.plusBlock(bw); Vector sq = accum.getRows()[0]; for (int k = 0; k < accum.getNumRows(); k++) { Vector btRow = accum.getRows()[k]; btKey.set((int) (key.get() * blockHeight + accum.getRowIndices()[k])); btValue.set(btRow); context.write(btKey, btValue);
@Override public void collect(Long rowIndex, Vector v) throws IOException { long blockKey = rowIndex / height; if (blockKey != currentBlockNum) { flushBlock(); if (block == null) { block = new SparseRowBlockWritable(100); } currentBlockNum = blockKey; } block.plusRow((int) (rowIndex % height), v); }
private void flushBlock() throws IOException { if (block == null || block.getNumRows() == 0) { return; } blockKeyW.set(currentBlockNum); delegate.collect(blockKeyW, block); block.clear(); }
} else { insertIntoPos(i, bOther.rowIndices[j], bOther.rows[j]); insertIntoPos(numRows, bOther.rowIndices[j], bOther.rows[j]);
InterruptedException { accum.clear(); for (SparseRowBlockWritable bw : values) { accum.plusBlock(bw); Vector sq = accum.getRows()[0]; for (int k = 0; k < accum.getNumRows(); k++) { Vector btRow = accum.getRows()[k]; btKey.set((int) (key.get() * blockHeight + accum.getRowIndices()[k])); btValue.set(btRow); context.write(btKey, btValue);
@Override protected void cleanup(Context context) throws IOException, InterruptedException { try { if (sqAccum != null) { /* * hack: we will output sq partial sums with index -1 for summation. */ SparseRowBlockWritable sbrw = new SparseRowBlockWritable(1); sbrw.plusRow(0, sqAccum); LongWritable lw = new LongWritable(-1); context.write(lw, sbrw); } } finally { IOUtils.close(closeables); } }
} else { insertIntoPos(i, bOther.rowIndices[j], bOther.rows[j]); insertIntoPos(numRows, bOther.rowIndices[j], bOther.rows[j]);