public void readyForFlush() { for (int i = 0; i < cellBlocks.length; i++) { cellBlocks[i].setLength(cellBlockBuffers[i].position()); } }
private void importFrom(DataInputStream in, ByteArray result) throws IOException { byte[] data = result.array(); int len = in.readInt(); in.read(data, 0, len); result.setLength(len); }
/** write data to given buffer, like serialize */ public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); Preconditions.checkNotNull(cols[c].array()); System.arraycopy(cols[c].array(), cols[c].offset(), buf.array(), buf.offset() + pos, cols[c].length()); pos += cols[c].length(); } buf.setLength(pos); }
private int buildKey(Cuboid cuboid, ByteArray[] splitBuffers) { RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid); int startIdx = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId int endIdx = startIdx + Long.bitCount(cuboid.getId()); int offset = 0; for (int i = startIdx; i < endIdx; i++) { System.arraycopy(splitBuffers[i].array(), splitBuffers[i].offset(), newKeyBodyBuf, offset, splitBuffers[i].length()); offset += splitBuffers[i].length(); } int fullKeySize = rowkeyEncoder.getBytesLength(); while (newKeyBuf.array().length < fullKeySize) { newKeyBuf = new ByteArray(newKeyBuf.length() * 2); } newKeyBuf.setLength(fullKeySize); rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf); return fullKeySize; }
@Override public GTRecord next() { for (int i = 0; i < colSpecs.size(); i++) { ColSpec spec = colSpecs.get(i); if (spec.cardinality > 0) { // dimension case long v = dist[i].next(); if (v < 0) { dist[i] = new Distribution(spec, parentLevelCount(i)); v = dist[i].next(); } ByteArray bytes = rec.get(i); assert bytes.length() == spec.length; BytesUtil.writeLong(v, bytes.array(), bytes.offset(), bytes.length()); } else { // measure case int len = spec.measureRandomizer.fillRandom(rand, rec.get(i).array(), 0); rec.get(i).setLength(len); } } counter++; return rec; }
private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); ByteArray columnC = record.get(c); if (columnC.array() != null) { System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length()); pos += columnC.length(); } else { int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c); Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue); pos += maxLength; } } buf.setLength(pos); }
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); GTInfo info = record.getInfo(); byte fill; int pos = 0; for (int i = 0; i < info.getPrimaryKey().trueBitCount(); i++) { int c = info.getPrimaryKey().trueBitAt(i); int colLength = info.getCodeSystem().maxCodeLength(c); if (record.get(c).array() != null) { fill = RowConstants.BYTE_ZERO; } else { fill = RowConstants.BYTE_ONE; } Arrays.fill(byteArray.array(), byteArray.offset() + pos, byteArray.offset() + pos + colLength, fill); pos += colLength; } byteArray.setLength(pos); //fill shard and cuboid fillHeader(buf); }
newKeyBuf = new ByteArray(newKeyBuf.length() * 2); newKeyBuf.setLength(fullKeySize);
public void readyForFlush() { for (int i = 0; i < cellBlocks.length; i++) { cellBlocks[i].setLength(cellBlockBuffers[i].position()); } }
private void importFrom(DataInputStream in, ByteArray result) throws IOException { byte[] data = result.array(); int len = in.readInt(); in.read(data, 0, len); result.setLength(len); }
/** write data to given buffer, like serialize */ public void exportColumns(ImmutableBitSet selectedCols, ByteArray buf) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); Preconditions.checkNotNull(cols[c].array()); System.arraycopy(cols[c].array(), cols[c].offset(), buf.array(), buf.offset() + pos, cols[c].length()); pos += cols[c].length(); } buf.setLength(pos); }
private int buildKey(Cuboid cuboid, ByteArray[] splitBuffers) { RowKeyEncoder rowkeyEncoder = rowKeyEncoderProvider.getRowkeyEncoder(cuboid); int startIdx = rowKeySplitter.getBodySplitOffset(); // skip shard and cuboidId int endIdx = startIdx + Long.bitCount(cuboid.getId()); int offset = 0; for (int i = startIdx; i < endIdx; i++) { System.arraycopy(splitBuffers[i].array(), splitBuffers[i].offset(), newKeyBodyBuf, offset, splitBuffers[i].length()); offset += splitBuffers[i].length(); } int fullKeySize = rowkeyEncoder.getBytesLength(); while (newKeyBuf.array().length < fullKeySize) { newKeyBuf = new ByteArray(newKeyBuf.length() * 2); } newKeyBuf.setLength(fullKeySize); rowkeyEncoder.encode(new ByteArray(newKeyBodyBuf, 0, offset), newKeyBuf); return fullKeySize; }
@Override public GTRecord next() { for (int i = 0; i < colSpecs.size(); i++) { ColSpec spec = colSpecs.get(i); if (spec.cardinality > 0) { // dimension case long v = dist[i].next(); if (v < 0) { dist[i] = new Distribution(spec, parentLevelCount(i)); v = dist[i].next(); } ByteArray bytes = rec.get(i); assert bytes.length() == spec.length; BytesUtil.writeLong(v, bytes.array(), bytes.offset(), bytes.length()); } else { // measure case int len = spec.measureRandomizer.fillRandom(rand, rec.get(i).array(), 0); rec.get(i).setLength(len); } } counter++; return rec; }
private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); ByteArray columnC = record.get(c); if (columnC.array() != null) { System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length()); pos += columnC.length(); } else { int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c); Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue); pos += maxLength; } } buf.setLength(pos); }
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); GTInfo info = record.getInfo(); byte fill; int pos = 0; for (int i = 0; i < info.getPrimaryKey().trueBitCount(); i++) { int c = info.getPrimaryKey().trueBitAt(i); int colLength = info.getCodeSystem().maxCodeLength(c); if (record.get(c).array() != null) { fill = RowConstants.BYTE_ZERO; } else { fill = RowConstants.BYTE_ONE; } Arrays.fill(byteArray.array(), byteArray.offset() + pos, byteArray.offset() + pos + colLength, fill); pos += colLength; } byteArray.setLength(pos); //fill shard and cuboid fillHeader(buf); }
newKeyBuf = new ByteArray(newKeyBuf.length() * 2); newKeyBuf.setLength(fullKeySize);