@Override public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange) { long startInclusive = (Long) segRange.start.v; long endExclusive = (Long) segRange.end.v; TblColRef partitionColumn = partDesc.getPartitionDateColumnRef(); String tableAlias = partitionColumn.getTableAlias(); String concatField = String.format(Locale.ROOT, "CONCAT(%s.YEAR,'-',%s.MONTH,'-',%s.DAY)", tableAlias, tableAlias, tableAlias); StringBuilder builder = new StringBuilder(); if (startInclusive > 0) { builder.append(concatField + " >= '" + DateFormat.formatToDateStr(startInclusive) + "' "); builder.append("AND "); } builder.append(concatField + " < '" + DateFormat.formatToDateStr(endExclusive) + "'"); return builder.toString(); } }
private static TblColRef getPartitionCol(IRealization realization) { String modelName = realization.getModel().getName(); DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv()) .getDataModelDesc(modelName); PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc(); Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!"); TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef(); Preconditions.checkArgument(partitionColRef != null, "getPartitionDateColumnRef for " + realization + " is null"); return partitionColRef; }
TblColRef col = partDesc.getPartitionDateColumnRef(); if (!groups.contains(col) && !singleValuesD.contains(col)) { logger.info("exactAggregation is false because cube is partitioned and %s is not on group by", col);
private String tsRangeToStr(long ts, PartitionDesc part) { String value; DataType partitionColType = part.getPartitionDateColumnRef().getType(); if (partitionColType.isDate()) { value = DateFormat.formatToDateStr(ts); } else if (partitionColType.isTimeFamily()) { value = DateFormat.formatToTimeWithoutMilliStr(ts); } else if (partitionColType.isStringFamily() || partitionColType.isIntegerFamily()) {//integer like 20160101 String partitionDateFormat = part.getPartitionDateFormat(); if (StringUtils.isEmpty(partitionDateFormat)) { value = "" + ts; } else { value = DateFormat.formatToDateStr(ts, partitionDateFormat); } } else { throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); } return value; }
private void updateSegment(CubeSegment segment) throws IOException { final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
private DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) { DataModelDesc model = seg.getModel(); PartitionDesc part = model.getPartitionDesc(); if (!part.isPartitioned()) return null; if (!col.equals(part.getPartitionDateColumnRef())) return null; // deduce the dim range from TSRange TSRange tsRange = seg.getTSRange(); if (tsRange.start.isMin || tsRange.end.isMax) return null; // DimensionRangeInfo cannot express infinite String min = tsRangeToStr(tsRange.start.v, part); String max = tsRangeToStr(tsRange.end.v - 1, part); // note the -1, end side is exclusive return new DimensionRangeInfo(min, max); }
@Override public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange) { long startInclusive = (Long) segRange.start.v; long endExclusive = (Long) segRange.end.v; TblColRef partitionDateColumn = partDesc.getPartitionDateColumnRef(); TblColRef partitionTimeColumn = partDesc.getPartitionTimeColumnRef(); StringBuilder builder = new StringBuilder(); if (partDesc.partitionColumnIsYmdInt()) { if (partitionTimeColumn == null) { buildSingleColumnRangeCondAsYmdInt(builder, partitionDateColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat()); } else { buildMultipleColumnRangeCondition(builder, partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), partDesc.getPartitionTimeFormat(), true); } } else if (partDesc.partitionColumnIsTimeMillis()) { buildSingleColumnRangeCondAsTimeMillis(builder, partitionDateColumn, startInclusive, endExclusive); } else if (partitionDateColumn != null && partitionTimeColumn == null) { buildSingleColumnRangeCondition(builder, partitionDateColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat()); } else if (partitionDateColumn == null && partitionTimeColumn != null) { buildSingleColumnRangeCondition(builder, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionTimeFormat()); } else if (partitionDateColumn != null && partitionTimeColumn != null) { buildMultipleColumnRangeCondition(builder, partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), partDesc.getPartitionTimeFormat(), false); } return builder.toString(); }
return partitionDesc.getPartitionDateColumnRef();
@Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING) { return; } if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) { context.addResult(ResultLevel.ERROR, "Must define a partition column."); return; } final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef(); boolean found = false; for (DimensionDesc dimensionDesc : cube.getDimensions()) { for (TblColRef dimCol : dimensionDesc.getColumnRefs()) { if (dimCol.equals(partitionCol)) { found = true; break; } } } if (found == false) { context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list."); return; } }
SegmentRange segRange = flatDesc.getSegRange(); if (segRange != null && !segRange.isInfinite()) { if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias) && (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
SegmentRange segRange = flatDesc.getSegRange(); if (segRange != null && !segRange.isInfinite()) { if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias) && (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
private void init(Collection<ColumnValueRange> andDimensionRanges) { int size = andDimensionRanges.size(); Map<TblColRef, String> startValues = Maps.newHashMapWithExpectedSize(size); Map<TblColRef, String> stopValues = Maps.newHashMapWithExpectedSize(size); Map<TblColRef, Set<String>> fuzzyValues = Maps.newHashMapWithExpectedSize(size); for (ColumnValueRange dimRange : andDimensionRanges) { TblColRef column = dimRange.getColumn(); startValues.put(column, dimRange.getBeginValue()); stopValues.put(column, dimRange.getEndValue()); fuzzyValues.put(column, dimRange.getEqualValues()); TblColRef partitionDateColumnRef = cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef(); if (column.equals(partitionDateColumnRef)) { initPartitionRange(dimRange); } } AbstractRowKeyEncoder encoder = AbstractRowKeyEncoder.createInstance(cubeSeg, cuboid); encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE); this.startKey = encoder.encode(startValues); encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE); // In order to make stopRow inclusive add a trailing 0 byte. #See // Scan.setStopRow(byte [] stopRow) this.stopKey = Bytes.add(encoder.encode(stopValues), ZERO_TAIL_BYTES); // restore encoder defaults for later reuse (note // AbstractRowKeyEncoder.createInstance() caches instances) encoder.setBlankByte(AbstractRowKeyEncoder.DEFAULT_BLANK_BYTE); // always fuzzy match cuboid ID to lock on the selected cuboid this.fuzzyKeys = buildFuzzyKeys(fuzzyValues); }
@Override public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange) { long startInclusive = (Long) segRange.start.v; long endExclusive = (Long) segRange.end.v; TblColRef partitionColumn = partDesc.getPartitionDateColumnRef(); String tableAlias = partitionColumn.getTableAlias(); String concatField = String.format(Locale.ROOT, "CONCAT(%s.YEAR,'-',%s.MONTH,'-',%s.DAY)", tableAlias, tableAlias, tableAlias); StringBuilder builder = new StringBuilder(); if (startInclusive > 0) { builder.append(concatField + " >= '" + DateFormat.formatToDateStr(startInclusive) + "' "); builder.append("AND "); } builder.append(concatField + " < '" + DateFormat.formatToDateStr(endExclusive) + "'"); return builder.toString(); } }
private static TblColRef getPartitionCol(IRealization realization) { String modelName = realization.getModel().getName(); DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv()) .getDataModelDesc(modelName); PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc(); Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!"); TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef(); Preconditions.checkArgument(partitionColRef != null, "getPartitionDateColumnRef for " + realization + " is null"); return partitionColRef; }
TblColRef col = partDesc.getPartitionDateColumnRef(); if (!groups.contains(col) && !singleValuesD.contains(col)) { logger.info("exactAggregation is false because cube is partitioned and %s is not on group by", col);
private String tsRangeToStr(long ts, PartitionDesc part) { String value; DataType partitionColType = part.getPartitionDateColumnRef().getType(); if (partitionColType.isDate()) { value = DateFormat.formatToDateStr(ts); } else if (partitionColType.isTimeFamily()) { value = DateFormat.formatToTimeWithoutMilliStr(ts); } else if (partitionColType.isStringFamily() || partitionColType.isIntegerFamily()) {//integer like 20160101 String partitionDateFormat = part.getPartitionDateFormat(); if (StringUtils.isEmpty(partitionDateFormat)) { value = "" + ts; } else { value = DateFormat.formatToDateStr(ts, partitionDateFormat); } } else { throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); } return value; }
private void updateSegment(CubeSegment segment) throws IOException { final TblColRef partitionCol = segment.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
private DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) { DataModelDesc model = seg.getModel(); PartitionDesc part = model.getPartitionDesc(); if (!part.isPartitioned()) return null; if (!col.equals(part.getPartitionDateColumnRef())) return null; // deduce the dim range from TSRange TSRange tsRange = seg.getTSRange(); if (tsRange.start.isMin || tsRange.end.isMax) return null; // DimensionRangeInfo cannot express infinite String min = tsRangeToStr(tsRange.start.v, part); String max = tsRangeToStr(tsRange.end.v - 1, part); // note the -1, end side is exclusive return new DimensionRangeInfo(min, max); }
@Override public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange) { long startInclusive = (Long) segRange.start.v; long endExclusive = (Long) segRange.end.v; TblColRef partitionDateColumn = partDesc.getPartitionDateColumnRef(); TblColRef partitionTimeColumn = partDesc.getPartitionTimeColumnRef(); StringBuilder builder = new StringBuilder(); if (partDesc.partitionColumnIsYmdInt()) { if (partitionTimeColumn == null) { buildSingleColumnRangeCondAsYmdInt(builder, partitionDateColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat()); } else { buildMultipleColumnRangeCondition(builder, partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), partDesc.getPartitionTimeFormat(), true); } } else if (partDesc.partitionColumnIsTimeMillis()) { buildSingleColumnRangeCondAsTimeMillis(builder, partitionDateColumn, startInclusive, endExclusive); } else if (partitionDateColumn != null && partitionTimeColumn == null) { buildSingleColumnRangeCondition(builder, partitionDateColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat()); } else if (partitionDateColumn == null && partitionTimeColumn != null) { buildSingleColumnRangeCondition(builder, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionTimeFormat()); } else if (partitionDateColumn != null && partitionTimeColumn != null) { buildMultipleColumnRangeCondition(builder, partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), partDesc.getPartitionTimeFormat(), false); } return builder.toString(); }
@Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING) { return; } if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) { context.addResult(ResultLevel.ERROR, "Must define a partition column."); return; } final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef(); boolean found = false; for (DimensionDesc dimensionDesc : cube.getDimensions()) { for (TblColRef dimCol : dimensionDesc.getColumnRefs()) { if (dimCol.equals(partitionCol)) { found = true; break; } } } if (found == false) { context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list."); return; } }