@Override public void write(long cuboidId, GTRecord record) throws IOException { if (lastCuboidId == null || !lastCuboidId.equals(cuboidId)) { if (lastCuboidId != null) { logger.info("Cuboid " + lastCuboidId + " has " + cuboidRowCount + " rows"); cuboidRowCount = 0; } // output another cuboid initVariables(cuboidId); lastCuboidId = cuboidId; } cuboidRowCount++; rowKeyEncoder.encode(record, record.getInfo().getPrimaryKey(), keyBuf); //output measures valueBuf.clear(); try { record.exportColumns(measureColumns, valueBuf); } catch (BufferOverflowException boe) { valueBuf = ByteBuffer.allocate((int) (record.sizeOf(measureColumns) * 1.5)); record.exportColumns(measureColumns, valueBuf); } outputKey.set(keyBuf, 0, keyBuf.length); outputValue.set(valueBuf.array(), 0, valueBuf.position()); writeAsKeyValue(outputKey, outputValue); }
private void decodeAndSetDimensions(GTRecord record) { for (int i = 0; i < gtDimsIdx.length; i++) { result[i] = record.decodeValue(gtDimsIdx[i]); } }
@Override public GTRecord next() { GTRecord x = input.next(); return new GTRecord(info, x.getInternal()); }
public ByteArray exportColumns(ImmutableBitSet selectedCols) { int len = sizeOf(selectedCols); ByteArray buf = ByteArray.allocate(len); exportColumns(selectedCols, buf); return buf; }
public InputConverter(GTInfo info, RecordConsumeBlockingQueueController<T> inputController) { this.info = info; this.inputController = inputController; this.record = new GTRecord(info); }
public Generator(long nRows) { this.nRows = nRows; this.rand = new Random(); rec = new GTRecord(info); dist = new Distribution[colSpecs.size()]; for (int i = 0; i < colSpecs.size(); i++) { ColSpec spec = colSpecs.get(i); rec.set(i, new ByteArray(spec.length)); dist[i] = new Distribution(spec, 0); } }
private void setMetricsValues(GTRecord record, Object[] metricsValues) { ImmutableBitSet metrics = getMetricsColumns(record); if (reuseMetricsSpace == null) { reuseMetricsSpace = new ByteArray(record.getInfo().getMaxColumnLength(metrics)); } record.setValues(metrics, reuseMetricsSpace, metricsValues); }
/** write data to given buffer, like serialize */ public void exportColumnBlock(int c, ByteBuffer buf) { exportColumns(info.colBlocks[c], buf); }
@Override public int compare(GTRecord a, GTRecord b) { assert a.getInfo() == b.getInfo(); int comp; ImmutableBitSet allColumns = a.getInfo().getAllColumns(); for (int i = 0; i < allColumns.trueBitCount(); i++) { int c = allColumns.trueBitAt(i); comp = comparator.compare(a.get(c), b.get(c)); if (comp != 0) return comp; } return 0; // equals } }
private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating cuboid {}", cuboidId); GTAggregateScanner scanner = prepareGTAggregationScanner(gridTable, parentId, cuboidId, aggregationColumns, measureColumns); GridTable newGridTable = newGridTableByCuboidID(cuboidId); GTBuilder builder = newGridTable.rebuild(); ImmutableBitSet allNeededColumns = aggregationColumns.or(measureColumns); GTRecord newRecord = new GTRecord(newGridTable.getInfo()); int count = 0; try { for (GTRecord record : scanner) { count++; for (int i = 0; i < allNeededColumns.trueBitCount(); i++) { int c = allNeededColumns.trueBitAt(i); newRecord.set(i, record.get(c)); } builder.write(newRecord); } } finally { scanner.close(); builder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", cuboidId, count, timeSpent); return updateCuboidResult(cuboidId, newGridTable, count, timeSpent, 0); }
private ImmutableBitSet getMetricsColumns(GTRecord record) { // metrics columns always come after dimension columns if (lastCuboidColumnCount == record.getInfo().getColumnCount()) return lastMetricsColumns; int to = record.getInfo().getColumnCount(); int from = to - reuseMetricsArray.length; lastCuboidColumnCount = record.getInfo().getColumnCount(); lastMetricsColumns = new ImmutableBitSet(from, to); return lastMetricsColumns; } }
public void append(GTRecord r) { // add record to block if (isEmpty()) { r.exportColumns(info.primaryKey, primaryKey); } for (int i = 0; i < info.colBlocks.length; i++) { r.exportColumnBlock(i, cellBlockBuffers[i]); } nRows++; }
/** toString for MemoryHungry Measure is expensive, please invoke carefully */ public String toString(ImmutableBitSet selectedColumns) { Object[] values = new Object[selectedColumns.cardinality()]; getValues(selectedColumns, values); return Arrays.toString(values); }
@Override public final void convert(String[] row, GTRecord record) { Object[] dimensions = kvBuilder.buildKey(row); Object[] metricsValues = buildValue(row); Object[] recordValues = new Object[dimensions.length + metricsValues.length]; System.arraycopy(dimensions, 0, recordValues, 0, dimensions.length); System.arraycopy(metricsValues, 0, recordValues, dimensions.length, metricsValues.length); record.setValues(recordValues); }
public GTStreamAggregateScanner(IGTScanner delegated, GTScanRequest scanRequest) { super(delegated); this.req = Preconditions.checkNotNull(scanRequest, "scanRequest"); this.keyComparator = GTRecord.getComparator(scanRequest.getAggrGroupBy()); }
private void serializeGTRecord(GTRecord gtRecord, ByteBuffer out) { ByteArray[] cols = gtRecord.getInternal(); BytesUtil.writeVInt(cols.length, out); for (ByteArray col : cols) { col.exportData(out); } } }
@Override public void write(GTRecord rec) throws IOException { assert info.getColumnBlockCount() == 2; rowkey.clear(); for (int i = 0; i < ID_LEN; i++) { rowkey.put((byte) 0); } rec.exportColumnBlock(0, rowkey); rowkey.flip(); value.clear(); rec.exportColumnBlock(1, value); value.flip(); Put put = new Put(rowkey); put.addImmutable(CF_B, ByteBuffer.wrap(COL_B), HConstants.LATEST_TIMESTAMP, value); table.mutate(put); }
SortMergedPartitionResultIterator(Iterator<PartitionResultIterator> iterators, GTInfo info, final Comparator<GTRecord> comparator) { this.iterators = iterators; this.record = new GTRecord(info); this.comparator = comparator; }
buffer.clear(); try { oneRecord.exportColumns(scanReq.getColumns(), buffer); } catch (BufferOverflowException boe) { buffer = ByteBuffer.allocate(oneRecord.sizeOf(scanReq.getColumns()) * 2); oneRecord.exportColumns(scanReq.getColumns(), buffer);