private void makeNext() { if (bitMapValues.isEmpty() == false) { addBitMapContainer(lastCol); } if (containers != null) { next = new Slice(info, lastShard, lastSliceTimestamp, containers); } lastSliceTimestamp = Long.MIN_VALUE; lastCol = -1; containers = null; bitMapValues.clear(); }
@Override public ConciseSet getBitMap(TblColRef col, Integer startId, Integer endId) { return slice.getColumnValueContainer(type.getColIndexByTblColRef(col)).getBitMap(startId,endId); }
public Collection<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> encodeKeyValue( Slice slice) { ArrayList<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> result = Lists .newArrayList(); ColumnValueContainer[] containers = slice.getColumnValueContainers(); for (int col = 0; col < containers.length; col++) { if (containers[col] instanceof BitMapContainer) { collectKeyValues(slice, col, (BitMapContainer) containers[col], result); } else if (containers[col] instanceof CompressedValueContainer) { collectKeyValues(slice, col, (CompressedValueContainer) containers[col], result); } else { throw new IllegalArgumentException("Unkown container class " + containers[col].getClass()); } } return result; }
private void collectKeyValues(Slice slice, int col, BitMapContainer container, // ArrayList<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> result) { List<ImmutableBytesWritable> values = container.toBytes(); for (int v = 0; v < values.size(); v++) { ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col, v); result.add(new Pair<ImmutableBytesWritable, ImmutableBytesWritable>( key, values.get(v))); } }
private IIRow collectKeyValues(Slice slice, int col, CompressedValueContainer container) { ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col); ImmutableBytesWritable value = container.toBytes(); final Dictionary<?> dictionary = slice.getLocalDictionaries() != null ? slice.getLocalDictionaries()[col] : null; if (dictionary == null) { return new IIRow(key, value, new ImmutableBytesWritable(BytesUtil.EMPTY_BYTE_ARRAY)); } else { ByteArray bytes = DictionarySerializer.serialize(dictionary); return new IIRow(key, value, new ImmutableBytesWritable(bytes.array(), bytes.offset(), bytes.length())); } }
Slice slice = new Slice(digest, curShard, curTimestamp, valueContainers); slice.setLocalDictionaries(localDictionaries); return slice;
private IIProtos.IIResponse getNonAggregatedResponse(Iterable<Slice> slices, CoprocessorFilter filter, CoprocessorRowType type) { IIProtos.IIResponse.Builder responseBuilder = IIProtos.IIResponse.newBuilder(); for (Slice slice : slices) { ConciseSet result = null; if (filter != null) { result = new BitMapFilterEvaluator(new SliceBitMapProvider(slice, type)).evaluate(filter.getFilter()); } Iterator<RawTableRecord> iterator = slice.iterateWithBitmap(result); while (iterator.hasNext()) { byte[] data = iterator.next().getBytes(); IIProtos.IIResponse.IIRow.Builder rowBuilder = IIProtos.IIResponse.IIRow.newBuilder().setColumns(ByteString.copyFrom(data)); responseBuilder.addRows(rowBuilder.build()); } } return responseBuilder.build(); }
public Iterator<RawTableRecord> iterateWithBitmap(final ConciseSet resultBitMap) { if (resultBitMap == null) { return this.iterator(); } else { final RawTableRecord rec = info.createTableRecordBytes();
@Override public int getRecordCount() { return this.slice.getRecordCount(); }
private Slice build(List<List<String>> table, final TableRecordInfo tableRecordInfo, Dictionary<?>[] localDictionary) { final Slice slice = sliceMaker.makeSlice(tableRecordInfo.getDigest(), Lists.transform(table, new Function<List<String>, TableRecord>() { @Nullable @Override public TableRecord apply(@Nullable List<String> input) { TableRecord result = tableRecordInfo.createTableRecord(); for (int i = 0; i < input.size(); i++) { result.setValueString(i, input.get(i)); } return result; } })); slice.setLocalDictionaries(localDictionary); return slice; } }
private void collectKeyValues(Slice slice, int col, CompressedValueContainer container, // ArrayList<Pair<ImmutableBytesWritable, ImmutableBytesWritable>> result) { ImmutableBytesWritable key = encodeKey(slice.getShard(), slice.getTimestamp(), col, -1); ImmutableBytesWritable value = container.toBytes(); result.add(new Pair<ImmutableBytesWritable, ImmutableBytesWritable>( key, value)); }
private IIProtos.IIResponse getAggregatedResponse(Iterable<Slice> slices, CoprocessorFilter filter, CoprocessorRowType type, CoprocessorProjector projector, EndpointAggregators aggregators) { EndpointAggregationCache aggCache = new EndpointAggregationCache(aggregators); IIProtos.IIResponse.Builder responseBuilder = IIProtos.IIResponse.newBuilder(); for (Slice slice : slices) { ConciseSet result = null; if (filter != null) { result = new BitMapFilterEvaluator(new SliceBitMapProvider(slice, type)).evaluate(filter.getFilter()); } Iterator<RawTableRecord> iterator = slice.iterateWithBitmap(result); while (iterator.hasNext()) { byte[] data = iterator.next().getBytes(); CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(data); MeasureAggregator[] bufs = aggCache.getBuffer(aggKey); aggregators.aggregate(bufs, data); aggCache.checkMemoryUsage(); } } byte[] metricBuffer = new byte[CoprocessorConstants.METRIC_SERIALIZE_BUFFER_SIZE]; for (Map.Entry<CoprocessorProjector.AggrKey, MeasureAggregator[]> entry : aggCache.getAllEntries()) { CoprocessorProjector.AggrKey aggrKey = entry.getKey(); IIProtos.IIResponse.IIRow.Builder rowBuilder = IIProtos.IIResponse.IIRow.newBuilder().setColumns(ByteString.copyFrom(aggrKey.get(), aggrKey.offset(), aggrKey.length())); int length = aggregators.serializeMetricValues(entry.getValue(), metricBuffer); rowBuilder.setMeasures(ByteString.copyFrom(metricBuffer, 0, length)); responseBuilder.addRows(rowBuilder.build()); } return responseBuilder.build(); }
public Iterator<RawTableRecord> iterateWithBitmap(final ConciseSet resultBitMap) { if (resultBitMap == null) { return this.iterator(); } else { final RawTableRecord rec = info.createTableRecordBytes();
private Slice doneSlice() { Slice r = null; if (nRecords > 0) { for (int i = 0; i < nColumns; i++) { containers[i].closeForChange(); } r = new Slice(info.getDigest(), shard, sliceTimestamp, containers); } // reset for next slice nRecords = 0; containers = new ColumnValueContainer[nColumns]; for (int i : info.getDescriptor().getBitmapColumns()) { containers[i] = new BitMapContainer(info.getDigest(), i); } for (int i : info.getDescriptor().getValueColumns()) { containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap); } for (int i : info.getDescriptor().getMetricsColumns()) { containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap); } return r; }
@Override public int getMaxValueId(TblColRef col) { return slice.getColumnValueContainer(type.getColIndexByTblColRef(col)).getMaxValueId(); } }
@Override public Collection<IIRow> encodeKeyValue(Slice slice) { ArrayList<IIRow> result = Lists.newArrayList(); ColumnValueContainer[] containers = slice.getColumnValueContainers(); for (int col = 0; col < containers.length; col++) { if (containers[col] instanceof CompressedValueContainer) { final IIRow row = collectKeyValues(slice, col, (CompressedValueContainer) containers[col]); result.add(row); } else { throw new IllegalArgumentException("Unknown container class " + containers[col].getClass()); } } return result; }
public Slice makeSlice(TableRecordInfoDigest digest, List<TableRecord> records) { Preconditions.checkArgument(records != null && !records.isEmpty(), "records cannot be empty"); Preconditions.checkArgument(records.size() <= nRecordsCap, "batch count cannot exceed " + nRecordsCap); sliceTimestamp = increaseSliceTimestamp(records.get(0).getTimestamp()); ColumnValueContainer[] containers = new ColumnValueContainer[nColumns]; for (int i : desc.getValueColumns()) { containers[i] = new CompressedValueContainer(digest, i, nRecordsCap); } for (int i : desc.getMetricsColumns()) { containers[i] = new CompressedValueContainer(digest, i, nRecordsCap); } for (TableRecord record : records) { for (int i = 0; i < nColumns; i++) { record.getValueBytes(i, temp); containers[i].append(temp); } } return new Slice(digest, shard, sliceTimestamp, containers); }
private Slice doneSlice() { Slice r = null; if (nRecords > 0) { for (int i = 0; i < nColumns; i++) { containers[i].closeForChange(); } r = new Slice(info.getDigest(), shard, sliceTimestamp, containers); } // reset for next slice nRecords = 0; containers = new ColumnValueContainer[nColumns]; // for (int i : info.getDescriptor().getBitmapColumns()) { // containers[i] = new CompressedValueContainer(info.getDigest(), i, // nRecordsCap); // } for (int i : info.getDescriptor().getValueColumns()) { containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap); } for (int i : info.getDescriptor().getMetricsColumns()) { containers[i] = new CompressedValueContainer(info.getDigest(), i, nRecordsCap); } return r; }