public void loadColumnsFromColumnBlocks(ImmutableBitSet[] selectedColumnBlocks, ImmutableBitSet selectedCols, ByteBuffer buf) { int pos = buf.position(); for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) { for (int i = 0; i < selectedColBlock.trueBitCount(); i++) { int c = selectedColBlock.trueBitAt(i); int len = info.codeSystem.codeLength(c, buf); if (selectedCols.get(c)) { cols[c].reset(buf.array(), buf.arrayOffset() + pos, len); } pos += len; buf.position(pos); } } }
@Override public MeasureAggregator<?>[] newMetricsAggregators(ImmutableBitSet columns, String[] aggrFunctions) { assert columns.trueBitCount() == aggrFunctions.length; MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length]; for (int i = 0; i < result.length; i++) { int col = columns.trueBitAt(i); result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col)); } // deal with holistic distinct count if (dependentMetricsMap != null) { for (Map.Entry<Integer, Integer> childEntry : dependentMetricsMap.entrySet()) { if (columns.get(childEntry.getKey())) { Integer parent = childEntry.getValue(); if (columns.get(parent) == false) throw new IllegalStateException(); int childIdx = columns.trueBitIndexOf(childEntry.getKey()); int parentIdx = columns.trueBitIndexOf(parent); result[childIdx].setDependentAggregator(result[parentIdx]); } } } return result; }
ByPassChecker(int aggregateBufferSizeLimit) { this.aggregateBufferSizeLimit = aggregateBufferSizeLimit; //init groupOffsetsInLastKey int p = 0; int idx = 0; this.groupOffsetsInLastKey = new int[groupBy.trueBitCount()]; for (int i = 0; i < dimensions.trueBitCount(); i++) { int c = dimensions.trueBitAt(i); int l = info.codeSystem.maxCodeLength(c); if (groupBy.get(c)) groupOffsetsInLastKey[idx++] = p; p += l; } }
/** * for each selected hbase column, it might contain values of multiple GT columns. * The mapping should be passed down to storage */ protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) { List<List<Integer>> ret = Lists.newArrayList(); int colBlkIndex = 1; int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount(); HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { int[] metricIndexes = hbaseColDesc.getMeasureIndex(); Integer[] gtIndexes = new Integer[metricIndexes.length]; for (int i = 0; i < gtIndexes.length; i++) { gtIndexes[i] = metricIndexes[i] + metricOffset; } ret.add(Arrays.asList(gtIndexes)); } colBlkIndex++; } } Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1); return ret; }
protected GTScanRange newScanRange(Collection<ColumnRange> andDimRanges) { GTRecord pkStart = new GTRecord(gtInfo); GTRecord pkEnd = new GTRecord(gtInfo); Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap(); for (ColumnRange range : andDimRanges) { int col = range.column.getColumnDesc().getZeroBasedIndex(); if (!gtInfo.getPrimaryKey().get(col)) continue; pkStart.set(col, range.begin); pkEnd.set(col, range.end); if (range.valueSet != null && !range.valueSet.isEmpty()) { fuzzyValues.put(col, range.valueSet); } } List<GTRecord> fuzzyKeys = buildFuzzyKeys(fuzzyValues); return new GTScanRange(pkStart, pkEnd, fuzzyKeys); }
@Override public Object getValue(TblColRef col) { int idx = col.getColumnDesc().getZeroBasedIndex(); return rtAggrMetrics.get(idx) ? DecimalUtil.toBigDecimal(next.getValue(idx)) : next.get(idx); } };
@Test public void testImmutableBitSet() { ImmutableBitSet x = new ImmutableBitSet(10, 100); ImmutableBitSet.serializer.serialize(x, buffer); buffer.flip(); ImmutableBitSet sx = ImmutableBitSet.serializer.deserialize(buffer); for (int i = 0; i < 10; i++) { Assert.assertFalse(sx.get(i)); } for (int i = 10; i < 100; i++) { Assert.assertTrue(sx.get(i)); } }
/** * prune untouched hbase columns */ protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) { List<Pair<byte[], byte[]>> result = Lists.newArrayList(); int colBlkIndex = 1; HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { byte[] byteFamily = Bytes.toBytes(familyDesc.getName()); for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier()); result.add(Pair.newPair(byteFamily, byteQualifier)); } colBlkIndex++; } } return result; }
private boolean[] createCompareMask() { int keyLength = 0; for (int i = 0; i < dimensions.trueBitCount(); i++) { int c = dimensions.trueBitAt(i); int l = info.codeSystem.maxCodeLength(c); keyLength += l; } boolean[] mask = new boolean[keyLength]; int p = 0; for (int i = 0; i < dimensions.trueBitCount(); i++) { int c = dimensions.trueBitAt(i); int l = info.codeSystem.maxCodeLength(c); boolean m = groupBy.get(c); for (int j = 0; j < l; j++) { mask[p++] = m; } } return mask; }
@Override public void init(GTInfo info) { this.info = info; ImmutableBitSet dDims = info.getDynamicDims(); this.serializers = new DataTypeSerializer[info.getColumnCount()]; for (int i = 0; i < serializers.length; i++) { DimensionEncoding dimEnc = i < dimEncs.length ? dimEncs[i] : null; if (dimEnc != null) { // for dimensions serializers[i] = dimEnc.asDataTypeSerializer(); } else { DataTypeSerializer dSerializer = DataTypeSerializer.create(info.getColumnType(i)); if (dDims != null && dDims.get(i)) { // for dynamic dimensions dSerializer = new DynamicDimSerializer(dSerializer); } else { // for measures } serializers[i] = dSerializer; } } }
public void loadColumnsFromColumnBlocks(ImmutableBitSet[] selectedColumnBlocks, ImmutableBitSet selectedCols, ByteBuffer buf) { int pos = buf.position(); for (ImmutableBitSet selectedColBlock : selectedColumnBlocks) { for (int i = 0; i < selectedColBlock.trueBitCount(); i++) { int c = selectedColBlock.trueBitAt(i); int len = info.codeSystem.codeLength(c, buf); if (selectedCols.get(c)) { cols[c].reset(buf.array(), buf.arrayOffset() + pos, len); } pos += len; buf.position(pos); } } }
@Override public MeasureAggregator<?>[] newMetricsAggregators(ImmutableBitSet columns, String[] aggrFunctions) { assert columns.trueBitCount() == aggrFunctions.length; MeasureAggregator<?>[] result = new MeasureAggregator[aggrFunctions.length]; for (int i = 0; i < result.length; i++) { int col = columns.trueBitAt(i); result[i] = MeasureAggregator.create(aggrFunctions[i], info.getColumnType(col)); } // deal with holistic distinct count if (dependentMetricsMap != null) { for (Map.Entry<Integer, Integer> childEntry : dependentMetricsMap.entrySet()) { if (columns.get(childEntry.getKey())) { Integer parent = childEntry.getValue(); if (columns.get(parent) == false) throw new IllegalStateException(); int childIdx = columns.trueBitIndexOf(childEntry.getKey()); int parentIdx = columns.trueBitIndexOf(parent); result[childIdx].setDependentAggregator(result[parentIdx]); } } } return result; }
/** * for each selected hbase column, it might contain values of multiple GT columns. * The mapping should be passed down to storage */ protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) { List<List<Integer>> ret = Lists.newArrayList(); int colBlkIndex = 1; int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount(); HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { int[] metricIndexes = hbaseColDesc.getMeasureIndex(); Integer[] gtIndexes = new Integer[metricIndexes.length]; for (int i = 0; i < gtIndexes.length; i++) { gtIndexes[i] = metricIndexes[i] + metricOffset; } ret.add(Arrays.asList(gtIndexes)); } colBlkIndex++; } } Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1); return ret; }
ByPassChecker(int aggregateBufferSizeLimit) { this.aggregateBufferSizeLimit = aggregateBufferSizeLimit; //init groupOffsetsInLastKey int p = 0; int idx = 0; this.groupOffsetsInLastKey = new int[groupBy.trueBitCount()]; for (int i = 0; i < dimensions.trueBitCount(); i++) { int c = dimensions.trueBitAt(i); int l = info.codeSystem.maxCodeLength(c); if (groupBy.get(c)) groupOffsetsInLastKey[idx++] = p; p += l; } }
@Override public Object getValue(TblColRef col) { int idx = col.getColumnDesc().getZeroBasedIndex(); return rtAggrMetrics.get(idx) ? DecimalUtil.toBigDecimal(next.getValue(idx)) : next.get(idx); } };
protected GTScanRange newScanRange(Collection<ColumnRange> andDimRanges) { GTRecord pkStart = new GTRecord(gtInfo); GTRecord pkEnd = new GTRecord(gtInfo); Map<Integer, Set<ByteArray>> fuzzyValues = Maps.newHashMap(); for (ColumnRange range : andDimRanges) { int col = range.column.getColumnDesc().getZeroBasedIndex(); if (!gtInfo.getPrimaryKey().get(col)) continue; pkStart.set(col, range.begin); pkEnd.set(col, range.end); if (range.valueSet != null && !range.valueSet.isEmpty()) { fuzzyValues.put(col, range.valueSet); } } List<GTRecord> fuzzyKeys = buildFuzzyKeys(fuzzyValues); return new GTScanRange(pkStart, pkEnd, fuzzyKeys); }
/** * prune untouched hbase columns */ protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) { List<Pair<byte[], byte[]>> result = Lists.newArrayList(); int colBlkIndex = 1; HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { byte[] byteFamily = Bytes.toBytes(familyDesc.getName()); for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier()); result.add(Pair.newPair(byteFamily, byteQualifier)); } colBlkIndex++; } } return result; }
private boolean[] createCompareMask() { int keyLength = 0; for (int i = 0; i < dimensions.trueBitCount(); i++) { int c = dimensions.trueBitAt(i); int l = info.codeSystem.maxCodeLength(c); keyLength += l; } boolean[] mask = new boolean[keyLength]; int p = 0; for (int i = 0; i < dimensions.trueBitCount(); i++) { int c = dimensions.trueBitAt(i); int l = info.codeSystem.maxCodeLength(c); boolean m = groupBy.get(c); for (int j = 0; j < l; j++) { mask[p++] = m; } } return mask; }
@Override public void init(GTInfo info) { this.info = info; ImmutableBitSet dDims = info.getDynamicDims(); this.serializers = new DataTypeSerializer[info.getColumnCount()]; for (int i = 0; i < serializers.length; i++) { DimensionEncoding dimEnc = i < dimEncs.length ? dimEncs[i] : null; if (dimEnc != null) { // for dimensions serializers[i] = dimEnc.asDataTypeSerializer(); } else { DataTypeSerializer dSerializer = DataTypeSerializer.create(info.getColumnType(i)); if (dDims != null && dDims.get(i)) { // for dynamic dimensions dSerializer = new DynamicDimSerializer(dSerializer); } else { // for measures } serializers[i] = dSerializer; } } }