private ImmutableBitSet getMetricsColumns(GTRecord record) { // metrics columns always come after dimension columns if (lastCuboidColumnCount == record.getInfo().getColumnCount()) return lastMetricsColumns; int to = record.getInfo().getColumnCount(); int from = to - reuseMetricsArray.length; lastCuboidColumnCount = record.getInfo().getColumnCount(); lastMetricsColumns = new ImmutableBitSet(from, to); return lastMetricsColumns; } }
private ImmutableBitSet getMetricsColumns(GTRecord record) { // metrics columns always come after dimension columns if (lastCuboidColumnCount == record.getInfo().getColumnCount()) return lastMetricsColumns; int to = record.getInfo().getColumnCount(); int from = to - reuseMetricsArray.length; lastCuboidColumnCount = record.getInfo().getColumnCount(); lastMetricsColumns = new ImmutableBitSet(from, to); return lastMetricsColumns; } }
@Override public int compare(GTRecord a, GTRecord b) { assert a.getInfo() == b.getInfo(); int comp; ImmutableBitSet allColumns = a.getInfo().getAllColumns(); for (int i = 0; i < allColumns.trueBitCount(); i++) { int c = allColumns.trueBitAt(i); comp = comparator.compare(a.get(c), b.get(c)); if (comp != 0) return comp; } return 0; // equals } }
/** * translate GTRecord format fuzzy keys to hbase expected format * @return */ private List<Pair<byte[], byte[]>> translateFuzzyKeys(List<GTRecord> fuzzyKeys) { if (fuzzyKeys == null || fuzzyKeys.isEmpty()) { return Collections.emptyList(); } List<Pair<byte[], byte[]>> ret = Lists.newArrayList(); for (GTRecord gtRecordFuzzyKey : fuzzyKeys) { byte[] hbaseFuzzyKey = fuzzyKeyEncoder.createBuf(); byte[] hbaseFuzzyMask = fuzzyMaskEncoder.createBuf(); fuzzyKeyEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyKey); fuzzyMaskEncoder.encode(gtRecordFuzzyKey, gtRecordFuzzyKey.getInfo().getPrimaryKey(), hbaseFuzzyMask); ret.add(Pair.newPair(hbaseFuzzyKey, hbaseFuzzyMask)); } return ret; }
private void setMetricsValues(GTRecord record, Object[] metricsValues) { ImmutableBitSet metrics = getMetricsColumns(record); if (reuseMetricsSpace == null) { reuseMetricsSpace = new ByteArray(record.getInfo().getMaxColumnLength(metrics)); } record.setValues(metrics, reuseMetricsSpace, metricsValues); }
private void setMetricsValues(GTRecord record, Object[] metricsValues) { ImmutableBitSet metrics = getMetricsColumns(record); if (reuseMetricsSpace == null) { reuseMetricsSpace = new ByteArray(record.getInfo().getMaxColumnLength(metrics)); } record.setValues(metrics, reuseMetricsSpace, metricsValues); }
@Override public void write(long cuboidId, GTRecord record) throws IOException { if (lastCuboidId == null || !lastCuboidId.equals(cuboidId)) { if (lastCuboidId != null) { logger.info("Cuboid " + lastCuboidId + " has " + cuboidRowCount + " rows"); cuboidRowCount = 0; } // output another cuboid initVariables(cuboidId); lastCuboidId = cuboidId; } cuboidRowCount++; rowKeyEncoder.encode(record, record.getInfo().getPrimaryKey(), keyBuf); //output measures valueBuf.clear(); try { record.exportColumns(measureColumns, valueBuf); } catch (BufferOverflowException boe) { valueBuf = ByteBuffer.allocate((int) (record.sizeOf(measureColumns) * 1.5)); record.exportColumns(measureColumns, valueBuf); } outputKey.set(keyBuf, 0, keyBuf.length); outputValue.set(valueBuf.array(), 0, valueBuf.position()); writeAsKeyValue(outputKey, outputValue); }
@Override public int compareTo(ResultMergeSlot o) { long cuboidComp = this.currentCuboidId - o.currentCuboidId; if (cuboidComp != 0) return cuboidComp < 0 ? -1 : 1; // note GTRecord.equals() don't work because the two GTRecord comes from different GridTable ImmutableBitSet pk = this.currentRecord.getInfo().getPrimaryKey(); for (int i = 0; i < pk.trueBitCount(); i++) { int c = pk.trueBitAt(i); int comp = this.currentRecord.get(c).compareTo(o.currentRecord.get(c)); if (comp != 0) return comp; } return 0; }
@Override public int compareTo(MergeSlot o) { long cuboidComp = this.currentCuboidId - o.currentCuboidId; if (cuboidComp != 0) return cuboidComp < 0 ? -1 : 1; // note GTRecord.equals() don't work because the two GTRecord comes from different GridTable ImmutableBitSet pk = this.currentRecord.getInfo().getPrimaryKey(); for (int i = 0; i < pk.trueBitCount(); i++) { int c = pk.trueBitAt(i); int comp = this.currentRecord.get(c).compareTo(o.currentRecord.get(c)); if (comp != 0) return comp; } return 0; }
private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) { final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks); LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid); byte[] start = encoder.createBuf(); byte[] end = encoder.createBuf(); encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE); encoder.encode(pkStart, pkStart.getInfo().getPrimaryKey(), start); encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE); encoder.encode(pkEnd, pkEnd.getInfo().getPrimaryKey(), end); byte[] temp = new byte[end.length + 1];//append extra 0 to the end key to make it inclusive while scanning System.arraycopy(end, 0, temp, 0, end.length); end = temp; List<Pair<byte[], byte[]>> hbaseFuzzyKeys = translateFuzzyKeys(fuzzyKeys); KylinConfig config = cubeSeg.getCubeDesc().getConfig(); int hbaseCaching = config.getHBaseScanCacheRows(); int hbaseMaxResultSize = config.getHBaseScanMaxResultSize(); // if (isMemoryHungry(selectedColBlocks)) // hbaseCaching /= 10; return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, hbaseCaching, hbaseMaxResultSize); }
private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); ByteArray columnC = record.get(c); if (columnC.array() != null) { System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length()); pos += columnC.length(); } else { int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c); Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue); pos += maxLength; } } buf.setLength(pos); }
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); GTInfo info = record.getInfo(); byte fill; int pos = 0; for (int i = 0; i < info.getPrimaryKey().trueBitCount(); i++) { int c = info.getPrimaryKey().trueBitAt(i); int colLength = info.getCodeSystem().maxCodeLength(c); if (record.get(c).array() != null) { fill = RowConstants.BYTE_ZERO; } else { fill = RowConstants.BYTE_ONE; } Arrays.fill(byteArray.array(), byteArray.offset() + pos, byteArray.offset() + pos + colLength, fill); pos += colLength; } byteArray.setLength(pos); //fill shard and cuboid fillHeader(buf); }
private ImmutableBitSet getMetricsColumns(GTRecord record) { // metrics columns always come after dimension columns if (lastCuboidColumnCount == record.getInfo().getColumnCount()) return lastMetricsColumns; int to = record.getInfo().getColumnCount(); int from = to - reuseMetricsArray.length; lastCuboidColumnCount = record.getInfo().getColumnCount(); lastMetricsColumns = new ImmutableBitSet(from, to); return lastMetricsColumns; } }
private ImmutableBitSet getMetricsColumns(GTRecord record) { // metrics columns always come after dimension columns if (lastCuboidColumnCount == record.getInfo().getColumnCount()) return lastMetricsColumns; int to = record.getInfo().getColumnCount(); int from = to - reuseMetricsArray.length; lastCuboidColumnCount = record.getInfo().getColumnCount(); lastMetricsColumns = new ImmutableBitSet(from, to); return lastMetricsColumns; } }
@Override public int compare(GTRecord a, GTRecord b) { assert a.getInfo() == b.getInfo(); int comp; ImmutableBitSet allColumns = a.getInfo().getAllColumns(); for (int i = 0; i < allColumns.trueBitCount(); i++) { int c = allColumns.trueBitAt(i); comp = comparator.compare(a.get(c), b.get(c)); if (comp != 0) return comp; } return 0; // equals } }
private void setMetricsValues(GTRecord record, Object[] metricsValues) { ImmutableBitSet metrics = getMetricsColumns(record); if (reuseMetricsSpace == null) { reuseMetricsSpace = new ByteArray(record.getInfo().getMaxColumnLength(metrics)); } record.setValues(metrics, reuseMetricsSpace, metricsValues); }
private void setMetricsValues(GTRecord record, Object[] metricsValues) { ImmutableBitSet metrics = getMetricsColumns(record); if (reuseMetricsSpace == null) { reuseMetricsSpace = new ByteArray(record.getInfo().getMaxColumnLength(metrics)); } record.setValues(metrics, reuseMetricsSpace, metricsValues); }
@Override public int compareTo(MergeSlot o) { long cuboidComp = this.currentCuboidId - o.currentCuboidId; if (cuboidComp != 0) return cuboidComp < 0 ? -1 : 1; // note GTRecord.equals() don't work because the two GTRecord comes from different GridTable ImmutableBitSet pk = this.currentRecord.getInfo().getPrimaryKey(); for (int i = 0; i < pk.trueBitCount(); i++) { int c = pk.trueBitAt(i); int comp = this.currentRecord.get(c).compareTo(o.currentRecord.get(c)); if (comp != 0) return comp; } return 0; }
@Override public int compareTo(ResultMergeSlot o) { long cuboidComp = this.currentCuboidId - o.currentCuboidId; if (cuboidComp != 0) return cuboidComp < 0 ? -1 : 1; // note GTRecord.equals() don't work because the two GTRecord comes from different GridTable ImmutableBitSet pk = this.currentRecord.getInfo().getPrimaryKey(); for (int i = 0; i < pk.trueBitCount(); i++) { int c = pk.trueBitAt(i); int comp = this.currentRecord.get(c).compareTo(o.currentRecord.get(c)); if (comp != 0) return comp; } return 0; }
private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); ByteArray columnC = record.get(c); if (columnC.array() != null) { System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length()); pos += columnC.length(); } else { int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c); Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue); pos += maxLength; } } buf.setLength(pos); }