public boolean isUsingDatetimeEncoding(int index) { return info.getCodeSystem().getDimEnc(index) instanceof AbstractDateDimEnc; }
private void validate() { if (info.getColumnCount() != colSpecs.size()) throw new IllegalArgumentException(); for (int i = 0; i < colSpecs.size(); i++) { ColSpec spec = colSpecs.get(i); if (info.getCodeSystem().maxCodeLength(i) < spec.length) throw new IllegalArgumentException(); } }
public boolean satisfyNone() { if (valueSet != null) { return valueSet.isEmpty(); } else if (begin.array() != null && end.array() != null) { return gtInfo.getCodeSystem().getComparator().compare(begin, end) > 0; } else { return false; } }
public static ByteArray enc(GTInfo info, int col, String value) { ByteBuffer buf = ByteBuffer.allocate(info.getMaxColumnLength()); info.getCodeSystem().encodeColumnValue(col, value, buf); return ByteArray.copyOf(buf.array(), buf.arrayOffset(), buf.position()); }
private ByteArray encodeTime(long ts, int index, int roundingFlag) { String value; DataType partitionColType = info.getColumnType(index); if (partitionColType.isDate()) { value = DateFormat.formatToDateStr(ts); } else if (partitionColType.isTimeFamily()) { value = DateFormat.formatToTimeWithoutMilliStr(ts); } else if (partitionColType.isStringFamily() || partitionColType.isIntegerFamily()) {//integer like 20160101 String partitionDateFormat = segment.getModel().getPartitionDesc().getPartitionDateFormat(); if (StringUtils.isEmpty(partitionDateFormat)) { value = "" + ts; } else { value = DateFormat.formatToDateStr(ts, partitionDateFormat); } } else { throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); } ByteBuffer buffer = ByteBuffer.allocate(info.getMaxColumnLength()); info.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer); return ByteArray.copyOf(buffer.array(), 0, buffer.position()); } }
private CompareTupleFilter eq(ColumnTupleFilter col, int... values) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN); r.addChild(col); List<ByteArray> list = Lists.newArrayList(); for (int v : values) { int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); list.add(bytes); } r.addChild(new ConstantTupleFilter(list)); return r; }
private CompareTupleFilter eq(ColumnTupleFilter col, int... values) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.IN); r.addChild(col); List<ByteArray> list = Lists.newArrayList(); for (int v : values) { int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); list.add(bytes); } r.addChild(new ConstantTupleFilter(list)); return r; }
AbstractStreamMergeIterator(Iterator<GTRecord> input) { this.input = Iterators.peekingIterator(input); this.codeSystem = req.getInfo().getCodeSystem(); this.dimensions = req.getDimensions(); this.metrics = req.getAggrMetrics(); this.metricFuncs = req.getAggrMetricsFuncs(); this.measureCodec = req.createMeasureCodec(); this.first = new GTRecord(req.getInfo()); }
IGTComparator comp = gtInfo.getCodeSystem().getComparator();
private void encodeDims(GTRecord record, ImmutableBitSet selectedCols, ByteArray buf, byte defaultValue) { int pos = 0; for (int i = 0; i < selectedCols.trueBitCount(); i++) { int c = selectedCols.trueBitAt(i); ByteArray columnC = record.get(c); if (columnC.array() != null) { System.arraycopy(record.get(c).array(), columnC.offset(), buf.array(), buf.offset() + pos, columnC.length()); pos += columnC.length(); } else { int maxLength = record.getInfo().getCodeSystem().maxCodeLength(c); Arrays.fill(buf.array(), buf.offset() + pos, buf.offset() + pos + maxLength, defaultValue); pos += maxLength; } } buf.setLength(pos); }
public static CompareTupleFilter getCompareTupleFilter(int col, Object value) { TblColRef colRef = gtInfo.colRef(col); ColumnTupleFilter colFilter = new ColumnTupleFilter(colRef); ByteArray space = new ByteArray(gtInfo.getCodeSystem().maxCodeLength(col)); gtInfo.getCodeSystem().encodeColumnValue(col, value, space.asBuffer()); ConstantTupleFilter constFilter = new ConstantTupleFilter(space); CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ); compareFilter.addChild(colFilter); compareFilter.addChild(constFilter); return compareFilter; }
/** * Construct GTScanRangePlanner with incomplete information. For UT only. */ public CubeScanRangePlanner(GTInfo info, TblColRef gtPartitionCol, TupleFilter gtFilter) { this.maxScanRanges = KylinConfig.getInstanceFromEnv().getQueryStorageVisitScanRangeMax(); this.maxFuzzyKeysPerSplit = KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeyMax(); this.maxFuzzyKeys = maxFuzzyKeysPerSplit * KylinConfig.getInstanceFromEnv().getQueryScanFuzzyKeySplitMax(); this.gtInfo = info; IGTComparator comp = gtInfo.getCodeSystem().getComparator(); //start key GTRecord compare to start key GTRecord this.rangeStartComparator = RecordComparators.getRangeStartComparator(comp); //stop key GTRecord compare to stop key GTRecord this.rangeEndComparator = RecordComparators.getRangeEndComparator(comp); //start key GTRecord compare to stop key GTRecord this.rangeStartEndComparator = RecordComparators.getRangeStartEndComparator(comp); this.gtFilter = gtFilter; }
private CompareTupleFilter gt(ColumnTupleFilter col, int v) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT); r.addChild(col); int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); r.addChild(new ConstantTupleFilter(bytes)); return r; }
private CompareTupleFilter gt(ColumnTupleFilter col, int v) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT); r.addChild(col); int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); r.addChild(new ConstantTupleFilter(bytes)); return r; }
@Override public void encode(GTRecord record, ImmutableBitSet keyColumns, byte[] buf) { ByteArray byteArray = new ByteArray(buf, getHeaderLength(), 0); GTInfo info = record.getInfo(); byte fill; int pos = 0; for (int i = 0; i < info.getPrimaryKey().trueBitCount(); i++) { int c = info.getPrimaryKey().trueBitAt(i); int colLength = info.getCodeSystem().maxCodeLength(c); if (record.get(c).array() != null) { fill = RowConstants.BYTE_ZERO; } else { fill = RowConstants.BYTE_ONE; } Arrays.fill(byteArray.array(), byteArray.offset() + pos, byteArray.offset() + pos + colLength, fill); pos += colLength; } byteArray.setLength(pos); //fill shard and cuboid fillHeader(buf); }
private static RawScan mockFullScan(GTInfo gtInfo, KylinConfig kylinConfig) { final List<Pair<byte[], byte[]>> selectedColumns = Lists.newArrayList(); selectedColumns.add(new Pair<>(FAM, COL_M)); int headerLength = RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN; int bodyLength = 0; ImmutableBitSet primaryKey = gtInfo.getPrimaryKey(); for (int i = 0; i < primaryKey.trueBitCount(); i++) { bodyLength += gtInfo.getCodeSystem().getDimEnc(primaryKey.trueBitAt(i)).getLengthOfEncoding(); } //Mock start key byte[] start = new byte[headerLength + bodyLength]; BytesUtil.writeShort((short) 0, start, 0, RowConstants.ROWKEY_SHARDID_LEN); System.arraycopy(Bytes.toBytes(baseCuboid), 0, start, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); //Mock end key byte[] end = new byte[headerLength + bodyLength + 1]; for (int i = 0; i < end.length - 1; i++) { end[i] = RowConstants.ROWKEY_UPPER_BYTE; } BytesUtil.writeShort((short) 0, end, 0, RowConstants.ROWKEY_SHARDID_LEN); System.arraycopy(Bytes.toBytes(baseCuboid), 0, end, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); //Mock fuzzy key List<Pair<byte[], byte[]>> fuzzyKeys = Collections.emptyList(); return new RawScan(start, end, selectedColumns, fuzzyKeys, kylinConfig.getHBaseScanCacheRows(), kylinConfig.getHBaseScanMaxResultSize()); }
private void validate() { if (info.getColumnCount() != colSpecs.size()) throw new IllegalArgumentException(); for (int i = 0; i < colSpecs.size(); i++) { ColSpec spec = colSpecs.get(i); if (info.getCodeSystem().maxCodeLength(i) < spec.length) throw new IllegalArgumentException(); } }
public boolean satisfyNone() { if (valueSet != null) { return valueSet.isEmpty(); } else if (begin.array() != null && end.array() != null) { return gtInfo.getCodeSystem().getComparator().compare(begin, end) > 0; } else { return false; } }
AbstractStreamMergeIterator(Iterator<GTRecord> input) { this.input = Iterators.peekingIterator(input); this.codeSystem = req.getInfo().getCodeSystem(); this.dimensions = req.getDimensions(); this.metrics = req.getAggrMetrics(); this.metricFuncs = req.getAggrMetricsFuncs(); this.measureCodec = req.createMeasureCodec(); this.first = new GTRecord(req.getInfo()); }
private CompareTupleFilter gt(ColumnTupleFilter col, int v) { CompareTupleFilter r = new CompareTupleFilter(FilterOperatorEnum.GT); r.addChild(col); int c = col.getColumn().getColumnDesc().getZeroBasedIndex(); int len = info.getCodeSystem().maxCodeLength(c); ByteArray bytes = new ByteArray(len); BytesUtil.writeLong(v, bytes.array(), bytes.offset(), len); r.addChild(new ConstantTupleFilter(bytes)); return r; }