@Override public int compare(GTRecord a, GTRecord b) { assert a.getInfo() == b.getInfo(); int comp; ImmutableBitSet allColumns = a.getInfo().getAllColumns(); for (int i = 0; i < allColumns.trueBitCount(); i++) { int c = allColumns.trueBitAt(i); comp = comparator.compare(a.get(c), b.get(c)); if (comp != 0) return comp; } return 0; // equals } }
@Override public Object getValue(TblColRef col) { return next.get(col.getColumnDesc().getZeroBasedIndex()); } };
@Override public Object getValue(TblColRef col) { int idx = col.getColumnDesc().getZeroBasedIndex(); return rtAggrMetrics.get(idx) ? DecimalUtil.toBigDecimal(next.getValue(idx)) : next.get(idx); } };
@Override public int compareTo(ResultMergeSlot o) { long cuboidComp = this.currentCuboidId - o.currentCuboidId; if (cuboidComp != 0) return cuboidComp < 0 ? -1 : 1; // note GTRecord.equals() don't work because the two GTRecord comes from different GridTable ImmutableBitSet pk = this.currentRecord.getInfo().getPrimaryKey(); for (int i = 0; i < pk.trueBitCount(); i++) { int c = pk.trueBitAt(i); int comp = this.currentRecord.get(c).compareTo(o.currentRecord.get(c)); if (comp != 0) return comp; } return 0; }
/** * @return true if should bypass this record */ boolean shouldByPass(GTRecord record) { if (dumps.size() > 0) { return false; //rare case: limit tends to be small, when limit is applied it's not likely to have dumps //TODO: what if bypass before dump happens? } Preconditions.checkState(aggBufMap.size() <= aggregateBufferSizeLimit); if (aggBufMap.size() == aggregateBufferSizeLimit) { Preconditions.checkNotNull(currentLastKey); for (int i = 0; i < groupBy.trueBitCount(); i++) { int c = groupBy.trueBitAt(i); ByteArray col = record.get(c); int compare = Bytes.compareTo(col.array(), col.offset(), col.length(), currentLastKey, groupOffsetsInLastKey[i], col.length()); if (compare > 0) { byPassCounter++; return true; } else if (compare < 0) { return false; } } } return false; }
@Override public Object getValue(TblColRef col) { int idx = col.getColumnDesc().getZeroBasedIndex(); return rtAggrMetrics.get(idx) ? DecimalUtil.toBigDecimal(next.getValue(idx)) : next.get(idx); } };
@Override public int compareTo(MergeSlot o) { long cuboidComp = this.currentCuboidId - o.currentCuboidId; if (cuboidComp != 0) return cuboidComp < 0 ? -1 : 1; // note GTRecord.equals() don't work because the two GTRecord comes from different GridTable ImmutableBitSet pk = this.currentRecord.getInfo().getPrimaryKey(); for (int i = 0; i < pk.trueBitCount(); i++) { int c = pk.trueBitAt(i); int comp = this.currentRecord.get(c).compareTo(o.currentRecord.get(c)); if (comp != 0) return comp; } return 0; }
@Override public GTRecord next() { for (int i = 0; i < colSpecs.size(); i++) { ColSpec spec = colSpecs.get(i); if (spec.cardinality > 0) { // dimension case long v = dist[i].next(); if (v < 0) { dist[i] = new Distribution(spec, parentLevelCount(i)); v = dist[i].next(); } ByteArray bytes = rec.get(i); assert bytes.length() == spec.length; BytesUtil.writeLong(v, bytes.array(), bytes.offset(), bytes.length()); } else { // measure case int len = spec.measureRandomizer.fillRandom(rand, rec.get(i).array(), 0); rec.get(i).setLength(len); } } counter++; return rec; }
private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); ByteArray columnC = record.get(c); if (columnC.array() != null) { System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length()); pos += columnC.length(); } else { int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c); Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue); pos += maxLength; } } buf.setLength(pos); }
private CuboidResult scanAndAggregateGridTable(GridTable gridTable, long parentId, long cuboidId, ImmutableBitSet aggregationColumns, ImmutableBitSet measureColumns) throws IOException { long startTime = System.currentTimeMillis(); logger.info("Calculating cuboid {}", cuboidId); GTAggregateScanner scanner = prepareGTAggregationScanner(gridTable, parentId, cuboidId, aggregationColumns, measureColumns); GridTable newGridTable = newGridTableByCuboidID(cuboidId); GTBuilder builder = newGridTable.rebuild(); ImmutableBitSet allNeededColumns = aggregationColumns.or(measureColumns); GTRecord newRecord = new GTRecord(newGridTable.getInfo()); int count = 0; try { for (GTRecord record : scanner) { count++; for (int i = 0; i < allNeededColumns.trueBitCount(); i++) { int c = allNeededColumns.trueBitAt(i); newRecord.set(i, record.get(c)); } builder.write(newRecord); } } finally { scanner.close(); builder.close(); } long timeSpent = System.currentTimeMillis() - startTime; logger.info("Cuboid {} has {} rows, build takes {}ms", cuboidId, count, timeSpent); return updateCuboidResult(cuboidId, newGridTable, count, timeSpent, 0); }
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); GTInfo info = record.getInfo(); byte fill; int pos = 0; for (int i = 0; i < info.getPrimaryKey().trueBitCount(); i++) { int c = info.getPrimaryKey().trueBitAt(i); int colLength = info.getCodeSystem().maxCodeLength(c); if (record.get(c).array() != null) { fill = RowConstants.BYTE_ZERO; } else { fill = RowConstants.BYTE_ONE; } Arrays.fill(byteArray.array(), byteArray.offset() + pos, byteArray.offset() + pos + colLength, fill); pos += colLength; } byteArray.setLength(pos); //fill shard and cuboid fillHeader(buf); }
for (int i = 0; i < allNeededColumns.trueBitCount(); i++) { int c = allNeededColumns.trueBitAt(i); newRecord.set(i, record.get(c));
@Override public int compare(GTRecord a, GTRecord b) { assert a.getInfo() == b.getInfo(); int comp; ImmutableBitSet allColumns = a.getInfo().getAllColumns(); for (int i = 0; i < allColumns.trueBitCount(); i++) { int c = allColumns.trueBitAt(i); comp = comparator.compare(a.get(c), b.get(c)); if (comp != 0) return comp; } return 0; // equals } }
public boolean[] checkCache(GTRecord record) { if (!enabled) return null; count++; // disable the cache if the hit rate is bad if (count == CHECKPOINT) { if ((double) hit / (double) count < HIT_RATE_THRESHOLD) { enabled = false; } } boolean match = count > 1; int p = 0; for (int i = 0; i < colsInFilter.trueBitCount(); i++) { int c = colsInFilter.trueBitAt(i); ByteArray col = record.get(c); if (match) { match = BytesUtil.compareBytes(col.array(), col.offset(), lastValues, p, col.length()) == 0; } if (!match) { System.arraycopy(col.array(), col.offset(), lastValues, p, col.length()); } p += col.length(); } if (match) { hit++; return lastResult; } else { return null; } }
@Override public int compareTo(ResultMergeSlot o) { long cuboidComp = this.currentCuboidId - o.currentCuboidId; if (cuboidComp != 0) return cuboidComp < 0 ? -1 : 1; // note GTRecord.equals() don't work because the two GTRecord comes from different GridTable ImmutableBitSet pk = this.currentRecord.getInfo().getPrimaryKey(); for (int i = 0; i < pk.trueBitCount(); i++) { int c = pk.trueBitAt(i); int comp = this.currentRecord.get(c).compareTo(o.currentRecord.get(c)); if (comp != 0) return comp; } return 0; }
@Override public Object getValue(TblColRef col) { return next.get(col.getColumnDesc().getZeroBasedIndex()); } };
@Override public int compareTo(MergeSlot o) { long cuboidComp = this.currentCuboidId - o.currentCuboidId; if (cuboidComp != 0) return cuboidComp < 0 ? -1 : 1; // note GTRecord.equals() don't work because the two GTRecord comes from different GridTable ImmutableBitSet pk = this.currentRecord.getInfo().getPrimaryKey(); for (int i = 0; i < pk.trueBitCount(); i++) { int c = pk.trueBitAt(i); int comp = this.currentRecord.get(c).compareTo(o.currentRecord.get(c)); if (comp != 0) return comp; } return 0; }
@Override public GTRecord next() { for (int i = 0; i < colSpecs.size(); i++) { ColSpec spec = colSpecs.get(i); if (spec.cardinality > 0) { // dimension case long v = dist[i].next(); if (v < 0) { dist[i] = new Distribution(spec, parentLevelCount(i)); v = dist[i].next(); } ByteArray bytes = rec.get(i); assert bytes.length() == spec.length; BytesUtil.writeLong(v, bytes.array(), bytes.offset(), bytes.length()); } else { // measure case int len = spec.measureRandomizer.fillRandom(rand, rec.get(i).array(), 0); rec.get(i).setLength(len); } } counter++; return rec; }
private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); ByteArray columnC = record.get(c); if (columnC.array() != null) { System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length()); pos += columnC.length(); } else { int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c); Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue); pos += maxLength; } } buf.setLength(pos); }
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); GTInfo info = record.getInfo(); byte fill; int pos = 0; for (int i = 0; i < info.getPrimaryKey().trueBitCount(); i++) { int c = info.getPrimaryKey().trueBitAt(i); int colLength = info.getCodeSystem().maxCodeLength(c); if (record.get(c).array() != null) { fill = RowConstants.BYTE_ZERO; } else { fill = RowConstants.BYTE_ONE; } Arrays.fill(byteArray.array(), byteArray.offset() + pos, byteArray.offset() + pos + colLength, fill); pos += colLength; } byteArray.setLength(pos); //fill shard and cuboid fillHeader(buf); }