String partCol = null; if (partitionDesc.isPartitioned()) { partCol = partitionDesc.getPartitionDateColumn();//tablename.colname if (partitionDesc.isPartitioned()) { SegmentRange segRange = flatDesc.getSegRange(); if (segRange != null && !segRange.isInfinite()) { if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias) && (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) { String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc, partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc, flatDesc.getSegment(), segRange), "`");
@Test public void testDateAndTimePartition() { PartitionDesc partitionDesc = new PartitionDesc(); TblColRef col1 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string"); partitionDesc.setPartitionDateColumnRef(col1); partitionDesc.setPartitionDateColumn(col1.getCanonicalName()); partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); TblColRef col2 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string"); partitionDesc.setPartitionTimeColumnRef(col2); partitionDesc.setPartitionTimeColumn(col2.getCanonicalName()); partitionDesc.setPartitionTimeFormat("H"); TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals( "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))", condition); }
public static DataModelDesc copy(DataModelDesc orig, DataModelDesc copy) { copy.config = orig.config; copy.name = orig.name; copy.isDraft = orig.isDraft; copy.owner = orig.owner; copy.description = orig.description; copy.rootFactTable = orig.rootFactTable; copy.joinTables = orig.joinTables; copy.dimensions = orig.dimensions; copy.metrics = orig.metrics; copy.filterCondition = orig.filterCondition; copy.capacity = orig.capacity; if (orig.getPartitionDesc() != null) { copy.partitionDesc = PartitionDesc.getCopyOf(orig.getPartitionDesc()); } copy.updateRandomUuid(); return copy; } }
public static PartitionDesc getPartitionDesc(String tableName) { PartitionDesc partitionDesc = new PartitionDesc(); partitionDesc.setPartitionDateColumn(tableName + "." + TimePropertyEnum.DAY_DATE.toString()); partitionDesc.setPartitionTimeColumn(tableName + "." + TimePropertyEnum.DAY_TIME.toString()); return partitionDesc; }
@Override public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange) { long startInclusive = (Long) segRange.start.v; long endExclusive = (Long) segRange.end.v; TblColRef partitionDateColumn = partDesc.getPartitionDateColumnRef(); TblColRef partitionTimeColumn = partDesc.getPartitionTimeColumnRef(); StringBuilder builder = new StringBuilder(); if (partDesc.partitionColumnIsYmdInt()) { if (partitionTimeColumn == null) { buildSingleColumnRangeCondAsYmdInt(builder, partitionDateColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat()); } else { buildMultipleColumnRangeCondition(builder, partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), partDesc.getPartitionTimeFormat(), true); } } else if (partDesc.partitionColumnIsTimeMillis()) { buildSingleColumnRangeCondAsTimeMillis(builder, partitionDateColumn, startInclusive, endExclusive); } else if (partitionDateColumn != null && partitionTimeColumn == null) { buildSingleColumnRangeCondition(builder, partitionDateColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat()); } else if (partitionDateColumn == null && partitionTimeColumn != null) { buildSingleColumnRangeCondition(builder, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionTimeFormat()); } else if (partitionDateColumn != null && partitionTimeColumn != null) { buildMultipleColumnRangeCondition(builder, partitionDateColumn, partitionTimeColumn, startInclusive, endExclusive, partDesc.getPartitionDateFormat(), partDesc.getPartitionTimeFormat(), false); } return builder.toString(); }
private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine) { final String sep = singleLine ? " " : "\n"; StringBuilder whereBuilder = new StringBuilder(); whereBuilder.append("WHERE 1=1"); DataModelDesc model = flatDesc.getDataModel(); if (StringUtils.isNotEmpty(model.getFilterCondition())) { String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, model.getFilterCondition(), "`"); whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug } if (flatDesc.getSegment() != null) { PartitionDesc partDesc = model.getPartitionDesc(); if (partDesc != null && partDesc.getPartitionDateColumn() != null) { SegmentRange segRange = flatDesc.getSegRange(); if (segRange != null && !segRange.isInfinite()) { whereBuilder.append(" AND ("); String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc, partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange), "`"); whereBuilder.append(quotedPartitionCond); whereBuilder.append(")" + sep); } } } sql.append(whereBuilder.toString()); }
public static PartitionDesc getCopyOf(PartitionDesc partitionDesc) { PartitionDesc newPartDesc = new PartitionDesc(); newPartDesc.setCubePartitionType(partitionDesc.getCubePartitionType()); newPartDesc.setPartitionDateColumn(partitionDesc.getPartitionDateColumn()); newPartDesc.setPartitionDateFormat(partitionDesc.getPartitionDateFormat()); newPartDesc.setPartitionDateStart(partitionDesc.getPartitionDateStart()); return newPartDesc; } }
if (partDesc.isPartitioned()) { TblColRef col = partDesc.getPartitionDateColumnRef(); if (!groups.contains(col) && !singleValuesD.contains(col)) { logger.info("exactAggregation is false because cube is partitioned and %s is not on group by", col);
public DataModelDesc createModelDesc(String projectName, DataModelDesc desc) throws IOException { aclEvaluate.checkProjectWritePermission(projectName); Message msg = MsgPicker.getMsg(); if (getDataModelManager().getDataModelDesc(desc.getName()) != null) { throw new BadRequestException(String.format(Locale.ROOT, msg.getDUPLICATE_MODEL_NAME(), desc.getName())); } String factTableName = desc.getRootFactTableName(); TableDesc tableDesc = getTableManager().getTableDesc(factTableName, projectName); if (tableDesc.getSourceType() == ISourceAware.ID_STREAMING && (desc.getPartitionDesc() == null || desc.getPartitionDesc().getPartitionDateColumn() == null)) { throw new IllegalArgumentException("Must define a partition column."); } DataModelDesc createdDesc = null; String owner = SecurityContextHolder.getContext().getAuthentication().getName(); createdDesc = getDataModelManager().createDataModelDesc(desc, projectName, owner); return createdDesc; }
private void updatePartitionDesc(CubeDesc oldModel, DataModelDesc dm) { CubePartitionDesc partition = oldModel.getCubePartitionDesc(); PartitionDesc newPartition = new PartitionDesc(); if (partition.getPartitionDateColumn() != null) { String partitionCol = partition.getPartitionDateColumn(); String[] tablecolumn = partitionCol.split("\\."); if (tablecolumn != null && tablecolumn.length == 2) { // pattern is <tablename>.<colname> String tableFullName = getMetadataManager().appendDBName(tablecolumn[0]); newPartition.setPartitionDateColumn(tableFullName + "." + tablecolumn[1]); } else { if (partitionCol.indexOf(".") < 0) { // pattern is <colname> partitionCol = dm.getFactTable() + "." + partitionCol; } newPartition.setPartitionDateColumn(partitionCol); } } // only append is supported newPartition.setCubePartitionType(PartitionDesc.PartitionType.APPEND); newPartition.setPartitionDateStart(partition.getPartitionDateStart()); dm.setPartitionDesc(newPartition); }
@Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING) { return; } if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) { context.addResult(ResultLevel.ERROR, "Must define a partition column."); return; } final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef(); boolean found = false; for (DimensionDesc dimensionDesc : cube.getDimensions()) { for (TblColRef dimCol : dimensionDesc.getColumnRefs()) { if (dimCol.equals(partitionCol)) { found = true; break; } } } if (found == false) { context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list."); return; } }
private void checkCubeIsPartitioned(CubeInstance cube) { if (cube.getDescriptor().getModel().getPartitionDesc().isPartitioned() == false) { throw new IllegalStateException( "there is no partition date column specified, only full build is supported"); } }
@Test public void testDatePartition() { PartitionDesc partitionDesc = new PartitionDesc(); TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string"); partitionDesc.setPartitionDateColumnRef(col); partitionDesc.setPartitionDateColumn(col.getCanonicalName()); partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23")); String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition); range = new TSRange(0L, 0L); condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals("1=0", condition); }
@Override public String buildDateRangeCondition(PartitionDesc partDesc, ISegment seg, SegmentRange segRange) { long startInclusive = (Long) segRange.start.v; long endExclusive = (Long) segRange.end.v; TblColRef partitionColumn = partDesc.getPartitionDateColumnRef(); String tableAlias = partitionColumn.getTableAlias(); String concatField = String.format(Locale.ROOT, "CONCAT(%s.YEAR,'-',%s.MONTH,'-',%s.DAY)", tableAlias, tableAlias, tableAlias); StringBuilder builder = new StringBuilder(); if (startInclusive > 0) { builder.append(concatField + " >= '" + DateFormat.formatToDateStr(startInclusive) + "' "); builder.append("AND "); } builder.append(concatField + " < '" + DateFormat.formatToDateStr(endExclusive) + "'"); return builder.toString(); } }
private String tsRangeToStr(long ts, PartitionDesc part) { String value; DataType partitionColType = part.getPartitionDateColumnRef().getType(); if (partitionColType.isDate()) { value = DateFormat.formatToDateStr(ts); } else if (partitionColType.isTimeFamily()) { value = DateFormat.formatToTimeWithoutMilliStr(ts); } else if (partitionColType.isStringFamily() || partitionColType.isIntegerFamily()) {//integer like 20160101 String partitionDateFormat = part.getPartitionDateFormat(); if (StringUtils.isEmpty(partitionDateFormat)) { value = "" + ts; } else { value = DateFormat.formatToDateStr(ts, partitionDateFormat); } } else { throw new RuntimeException("Type " + partitionColType + " is not valid partition column type"); } return value; }
@Test public void testTimePartition() { PartitionDesc partitionDesc = new PartitionDesc(); TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string"); partitionDesc.setPartitionTimeColumnRef(col); partitionDesc.setPartitionTimeColumn(col.getCanonicalName()); partitionDesc.setPartitionTimeFormat("HH"); TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition); }
public static PartitionDesc getCopyOf(PartitionDesc orig) { PartitionDesc ret = new PartitionDesc(); ret.partitionDateColumn = orig.partitionDateColumn; ret.partitionTimeColumn = orig.partitionTimeColumn; ret.partitionDateStart = orig.partitionDateStart; //Deprecated ret.partitionDateFormat = orig.partitionDateFormat; ret.partitionTimeFormat = orig.partitionTimeFormat; ret.partitionType = orig.partitionType; ret.partitionConditionBuilderClz = orig.partitionConditionBuilderClz; return ret; }
@Override public String buildDateRangeCondition(PartitionDesc partDesc, long startInclusive, long endExclusive, Map<String, String> tableAlias) { String partitionColumnName = partDesc.getPartitionDateColumn(); // convert to use table alias int indexOfDot = partitionColumnName.lastIndexOf("."); if (indexOfDot > 0) { String partitionTableName = partitionColumnName.substring(0, indexOfDot); if (tableAlias != null && tableAlias.containsKey(partitionTableName)) partitionColumnName = tableAlias.get(partitionTableName) + partitionColumnName.substring(indexOfDot); } StringBuilder builder = new StringBuilder(); if (startInclusive > 0) { builder.append(partitionColumnName + " >= '" + DateFormat.formatToDateStr(startInclusive, partDesc.getPartitionDateFormat()) + "' "); builder.append("AND "); } builder.append(partitionColumnName + " < '" + DateFormat.formatToDateStr(endExclusive, partDesc.getPartitionDateFormat()) + "'"); return builder.toString(); }
@Test public void testBasics() throws Exception { CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_without_slr_ready"); CubeDesc desc = cube.getDescriptor(); //System.out.println(JsonUtil.writeValueAsIndentString(desc)); String signature = desc.calculateSignature(); desc.getModel().getPartitionDesc().setPartitionDateColumn("test_column"); assertTrue(signature.equals(desc.calculateSignature())); }
private long calculateStartDateForAppendSegment(CubeInstance cube) { List<CubeSegment> existing = cube.getSegments(); if (existing.isEmpty()) { return cube.getDescriptor().getModel().getPartitionDesc().getPartitionDateStart(); } else { return existing.get(existing.size() - 1).getDateRangeEnd(); } }