public static PartitionDesc getCopyOf(PartitionDesc orig) { PartitionDesc ret = new PartitionDesc(); ret.partitionDateColumn = orig.partitionDateColumn; ret.partitionTimeColumn = orig.partitionTimeColumn; ret.partitionDateStart = orig.partitionDateStart; //Deprecated ret.partitionDateFormat = orig.partitionDateFormat; ret.partitionTimeFormat = orig.partitionTimeFormat; ret.partitionType = orig.partitionType; ret.partitionConditionBuilderClz = orig.partitionConditionBuilderClz; return ret; }
public static PartitionDesc getPartitionDesc(String tableName) { PartitionDesc partitionDesc = new PartitionDesc(); partitionDesc.setPartitionDateColumn(tableName + "." + TimePropertyEnum.DAY_DATE.toString()); partitionDesc.setPartitionTimeColumn(tableName + "." + TimePropertyEnum.DAY_TIME.toString()); return partitionDesc; }
@Test public void testTimePartition() { PartitionDesc partitionDesc = new PartitionDesc(); TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string"); partitionDesc.setPartitionTimeColumnRef(col); partitionDesc.setPartitionTimeColumn(col.getCanonicalName()); partitionDesc.setPartitionTimeFormat("HH"); TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition); }
@Test public void testDatePartition() { PartitionDesc partitionDesc = new PartitionDesc(); TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string"); partitionDesc.setPartitionDateColumnRef(col); partitionDesc.setPartitionDateColumn(col.getCanonicalName()); partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23")); String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition); range = new TSRange(0L, 0L); condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals("1=0", condition); }
@Test public void testDateAndTimePartition() { PartitionDesc partitionDesc = new PartitionDesc(); TblColRef col1 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string"); partitionDesc.setPartitionDateColumnRef(col1); partitionDesc.setPartitionDateColumn(col1.getCanonicalName()); partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); TblColRef col2 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string"); partitionDesc.setPartitionTimeColumnRef(col2); partitionDesc.setPartitionTimeColumn(col2.getCanonicalName()); partitionDesc.setPartitionTimeFormat("H"); TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals( "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))", condition); }
@Test public void testAppendNonPartitioned() throws IOException { CubeManager mgr = mgr(); CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_empty"); // override partition desc cube.getModel().setPartitionDesc(new PartitionDesc()); // first append, creates a new & single segment CubeSegment seg = mgr.appendSegment(cube); assertEquals(new TSRange(0L, Long.MAX_VALUE), seg.getTSRange()); assertEquals(new TSRange(0L, Long.MAX_VALUE), seg.getSegRange()); assertEquals(0, cube.getSegments().size()); // older cube not changed cube = mgr.getCube(cube.getName()); assertEquals(1, cube.getSegments().size()); // the updated cube // second append, throw IllegalStateException because the first segment is not built try { mgr.appendSegment(cube); fail(); } catch (IllegalStateException ex) { // good } }
@Test public void testAppendNonPartitioned2() throws IOException { CubeManager mgr = mgr(); CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_ready"); // override partition desc cube.getModel().setPartitionDesc(new PartitionDesc()); // assert one ready segment assertEquals(1, cube.getSegments().size()); CubeSegment seg = cube.getSegments(SegmentStatusEnum.READY).get(0); assertEquals(SegmentStatusEnum.READY, seg.getStatus()); // append again, for non-partitioned cube, it becomes a full refresh CubeSegment seg2 = mgr.appendSegment(cube); assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getTSRange()); assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getSegRange()); assertEquals(1, cube.getSegments().size()); // older cube not changed cube = mgr.getCube(cube.getName()); assertEquals(2, cube.getSegments().size()); // the updated cube // non-partitioned cannot merge, throw exception try { mgr.mergeSegments(cube, null, new SegmentRange(0L, Long.MAX_VALUE), false); fail(); } catch (IllegalStateException ex) { // good } }
private void updatePartitionDesc(CubeDesc oldModel, DataModelDesc dm) { CubePartitionDesc partition = oldModel.getCubePartitionDesc(); PartitionDesc newPartition = new PartitionDesc(); if (partition.getPartitionDateColumn() != null) { String partitionCol = partition.getPartitionDateColumn(); String[] tablecolumn = partitionCol.split("\\."); if (tablecolumn != null && tablecolumn.length == 2) { // pattern is <tablename>.<colname> String tableFullName = getMetadataManager().appendDBName(tablecolumn[0]); newPartition.setPartitionDateColumn(tableFullName + "." + tablecolumn[1]); } else { if (partitionCol.indexOf(".") < 0) { // pattern is <colname> partitionCol = dm.getFactTable() + "." + partitionCol; } newPartition.setPartitionDateColumn(partitionCol); } } // only append is supported newPartition.setCubePartitionType(PartitionDesc.PartitionType.APPEND); newPartition.setPartitionDateStart(partition.getPartitionDateStart()); dm.setPartitionDesc(newPartition); }
public static PartitionDesc getCopyOf(PartitionDesc orig) { PartitionDesc ret = new PartitionDesc(); ret.partitionDateColumn = orig.partitionDateColumn; ret.partitionTimeColumn = orig.partitionTimeColumn; ret.partitionDateStart = orig.partitionDateStart; //Deprecated ret.partitionDateFormat = orig.partitionDateFormat; ret.partitionTimeFormat = orig.partitionTimeFormat; ret.partitionType = orig.partitionType; ret.partitionConditionBuilderClz = orig.partitionConditionBuilderClz; return ret; }
public static PartitionDesc getPartitionDesc(String tableName) { PartitionDesc partitionDesc = new PartitionDesc(); partitionDesc.setPartitionDateColumn(tableName + "." + TimePropertyEnum.DAY_DATE.toString()); partitionDesc.setPartitionTimeColumn(tableName + "." + TimePropertyEnum.DAY_TIME.toString()); return partitionDesc; }
public static PartitionDesc getCopyOf(PartitionDesc partitionDesc) { PartitionDesc newPartDesc = new PartitionDesc(); newPartDesc.setCubePartitionType(partitionDesc.getCubePartitionType()); newPartDesc.setPartitionDateColumn(partitionDesc.getPartitionDateColumn()); newPartDesc.setPartitionDateFormat(partitionDesc.getPartitionDateFormat()); newPartDesc.setPartitionDateStart(partitionDesc.getPartitionDateStart()); return newPartDesc; } }
private void updatePartitionDesc(CubeDesc oldModel, DataModelDesc dm) { CubePartitionDesc partition = oldModel.getCubePartitionDesc(); PartitionDesc newPartition = new PartitionDesc(); if (partition.getPartitionDateColumn() != null) { String partitionCol = partition.getPartitionDateColumn(); String[] tablecolumn = partitionCol.split("\\."); if (tablecolumn != null && tablecolumn.length == 2) { // pattern is <tablename>.<colname> String tableFullName = getMetadataManager().appendDBName(tablecolumn[0]); newPartition.setPartitionDateColumn(tableFullName + "." + tablecolumn[1]); } else { if (partitionCol.indexOf(".") < 0) { // pattern is <colname> partitionCol = dm.getFactTable() + "." + partitionCol; } newPartition.setPartitionDateColumn(partitionCol); } } // only append is supported newPartition.setCubePartitionType(PartitionDesc.PartitionType.APPEND); newPartition.setPartitionDateStart(partition.getPartitionDateStart()); dm.setPartitionDesc(newPartition); }