/** * Build/Rebuild a cube segment */ @RequestMapping(value = "/{cubeName}/rebuild", method = { RequestMethod.PUT }, produces = { "application/json" }) @ResponseBody public JobInstance rebuild(@PathVariable String cubeName, @RequestBody JobBuildRequest req) { return buildInternal(cubeName, new TSRange(req.getStartTime(), req.getEndTime()), null, null, null, req.getBuildType(), req.isForce() || req.isForceMergeEmptySegment()); }
private void run(String cubeName, long startDate, long endDate, String buildType) throws IOException, JobException { CubeInstance cube = cubeManager.getCube(cubeName); Preconditions.checkArgument(cube != null, "Cube named " + cubeName + " does not exist!!!"); CubeBuildTypeEnum buildTypeEnum = CubeBuildTypeEnum.valueOf(buildType); Preconditions.checkArgument(buildTypeEnum != null, "Build type named " + buildType + " does not exist!!!"); submitJob(cube, new TSRange(startDate, endDate), buildTypeEnum, false, "SYSTEM"); }
@Override public void run() { try { Random rand = new Random(); for (int i = 0; i < updatesPerCube; i++) { CubeManager mgr = CubeManager.getInstance(config); CubeInstance cube = mgr.getCube(cubeName); mgr.appendSegment(cube, new TSRange((long) i, (long) i + 1)); Thread.sleep(rand.nextInt(1)); } } catch (Exception ex) { logger.error("update thread ", ex); exceptions.add(ex); } } };
CubeSegment appendSegment(CubeInstance cube, TSRange tsRange, SegmentRange segRange, Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd) throws IOException { CubeInstance cubeCopy = cube.latestCopyForWrite(); // get a latest copy checkInputRanges(tsRange, segRange); // fix start/end a bit if (cubeCopy.getModel().getPartitionDesc().isPartitioned()) { // if missing start, set it to where last time ends if (tsRange != null && tsRange.start.v == 0) { CubeDesc cubeDesc = cubeCopy.getDescriptor(); CubeSegment last = cubeCopy.getLastSegment(); if (last == null) tsRange = new TSRange(cubeDesc.getPartitionDateStart(), tsRange.end.v); else if (!last.isOffsetCube()) tsRange = new TSRange(last.getTSRange().end.v, tsRange.end.v); } } else { // full build tsRange = null; segRange = null; } CubeSegment newSegment = newSegment(cubeCopy, tsRange, segRange); newSegment.setSourcePartitionOffsetStart(sourcePartitionOffsetStart); newSegment.setSourcePartitionOffsetEnd(sourcePartitionOffsetEnd); validateNewSegments(cubeCopy, newSegment); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToAddSegs(newSegment); updateCube(update); return newSegment; }
@Test public void testAllowGap() throws IOException { CubeManager mgr = mgr(); CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_left_join_empty"); // no segment at first assertEquals(0, cube.getSegments().size()); // append the first CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L)); cube = readySegment(cube, seg1); assertEquals(1, cube.getSegments().size()); // append the third CubeSegment seg3 = mgr.appendSegment(cube, new TSRange(2000L, 3000L)); cube = readySegment(cube, seg3); assertEquals(2, cube.getSegments().size()); // reject overlap try { mgr.appendSegment(cube, new TSRange(1000L, 2500L)); fail(); } catch (IllegalStateException ex) { // good } // append the second CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 2000L)); cube = readySegment(cube, seg2); assertEquals(3, cube.getSegments().size()); }
@Test public void testAppendNonPartitioned2() throws IOException { CubeManager mgr = mgr(); CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_ready"); // override partition desc cube.getModel().setPartitionDesc(new PartitionDesc()); // assert one ready segment assertEquals(1, cube.getSegments().size()); CubeSegment seg = cube.getSegments(SegmentStatusEnum.READY).get(0); assertEquals(SegmentStatusEnum.READY, seg.getStatus()); // append again, for non-partitioned cube, it becomes a full refresh CubeSegment seg2 = mgr.appendSegment(cube); assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getTSRange()); assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getSegRange()); assertEquals(1, cube.getSegments().size()); // older cube not changed cube = mgr.getCube(cube.getName()); assertEquals(2, cube.getSegments().size()); // the updated cube // non-partitioned cannot merge, throw exception try { mgr.mergeSegments(cube, null, new SegmentRange(0L, Long.MAX_VALUE), false); fail(); } catch (IllegalStateException ex) { // good } }
@Test public void testGenSqoopCmd_WithLookupShardBy() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\"")); source.close(); }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeManager = CubeManager.getInstance(context.getConfig()); final CubeInstance cubeCopy = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite(); final String segmentId = CubingExecutableUtil.getSegmentId(this.getParams()); final CubeSegment segCopy = cubeCopy.getSegmentById(segmentId); Preconditions.checkNotNull(segCopy, "Cube segment '" + segmentId + "' not found."); Segments<CubeSegment> mergingSegs = cubeCopy.getMergingSegments(segCopy); Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment not exist."); Collections.sort(mergingSegs); final CubeSegment first = mergingSegs.get(0); final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1); segCopy.setSegRange(new SegmentRange(first.getSegRange().start, last.getSegRange().end)); segCopy.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart()); segCopy.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd()); segCopy.setTSRange(new TSRange(mergingSegs.getTSStart(), mergingSegs.getTSEnd())); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToUpdateSegs(segCopy); try { cubeManager.updateCube(update); return ExecuteResult.createSucceed(); } catch (IOException e) { logger.error("fail to update cube segment offset", e); return ExecuteResult.createError(e); } }
@Test public void testGenSqoopCmd_Partition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue(cmd.contains( "--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"")); source.close(); }
@Test public void testAppendNonPartitioned() throws IOException { CubeManager mgr = mgr(); CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_empty"); // override partition desc cube.getModel().setPartitionDesc(new PartitionDesc()); // first append, creates a new & single segment CubeSegment seg = mgr.appendSegment(cube); assertEquals(new TSRange(0L, Long.MAX_VALUE), seg.getTSRange()); assertEquals(new TSRange(0L, Long.MAX_VALUE), seg.getSegRange()); assertEquals(0, cube.getSegments().size()); // older cube not changed cube = mgr.getCube(cube.getName()); assertEquals(1, cube.getSegments().size()); // the updated cube // second append, throw IllegalStateException because the first segment is not built try { mgr.appendSegment(cube); fail(); } catch (IllegalStateException ex) { // good } }
@Test public void testGenSqoopCmd_NoPartition() throws IOException { ISource source = SourceManager.getSource(new JdbcSourceAware()); IMRInput input = source.adaptToBuildEngine(IMRInput.class); Assert.assertNotNull(input); CubeManager cubeManager = CubeManager.getInstance(getTestConfig()); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube"); CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()), new SegmentRange.TSRange(0L, Long.MAX_VALUE)); CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg); JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input .getBatchCubingInputSide(flatDesc); AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp", cubeDesc.getName()); Assert.assertNotNull(executable); String cmd = executable.getParam("cmd"); Assert.assertTrue(cmd.contains("org.h2.Driver")); Assert.assertTrue( cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")\n" + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\"")); source.close(); }
private CubeSegment newSegment(CubeInstance cube, TSRange tsRange, SegmentRange segRange) { DataModelDesc modelDesc = cube.getModel(); CubeSegment segment = new CubeSegment(); segment.setUuid(RandomUtil.randomUUID().toString()); segment.setName(CubeSegment.makeSegmentName(tsRange, segRange, modelDesc)); segment.setCreateTimeUTC(System.currentTimeMillis()); segment.setCubeInstance(cube); // let full build range be backward compatible if (tsRange == null && segRange == null) tsRange = new TSRange(0L, Long.MAX_VALUE); segment.setTSRange(tsRange); segment.setSegRange(segRange); segment.setStatus(SegmentStatusEnum.NEW); segment.setStorageLocationIdentifier(generateStorageLocation()); segment.setCubeInstance(cube); segment.validate(); return segment; }
@Test public void testDateAndTimePartition() { PartitionDesc partitionDesc = new PartitionDesc(); TblColRef col1 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string"); partitionDesc.setPartitionDateColumnRef(col1); partitionDesc.setPartitionDateColumn(col1.getCanonicalName()); partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); TblColRef col2 = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string"); partitionDesc.setPartitionTimeColumnRef(col2); partitionDesc.setPartitionTimeColumn(col2.getCanonicalName()); partitionDesc.setPartitionTimeFormat("H"); TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals( "((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-22' AND UNKNOWN_ALIAS.HOUR_COLUMN >= '0') OR (UNKNOWN_ALIAS.DATE_COLUMN > '2016-02-22')) AND ((UNKNOWN_ALIAS.DATE_COLUMN = '2016-02-23' AND UNKNOWN_ALIAS.HOUR_COLUMN < '1') OR (UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'))", condition); }
@Test public void testAutoMergeNormal() throws Exception { CubeManager mgr = CubeManager.getInstance(getTestConfig()); CubeInstance cube = mgr.getCube("test_kylin_cube_with_slr_empty").latestCopyForWrite(); cube.getDescriptor().setAutoMergeTimeRanges(new long[] { 2000, 6000 }); mgr.updateCube(new CubeUpdate(cube)); assertTrue(cube.needAutoMerge()); // no segment at first assertEquals(0, cube.getSegments().size()); // append first CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L), null, null, null); mgr.updateCubeSegStatus(seg1, SegmentStatusEnum.READY); CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 2000L), null, null, null); mgr.updateCubeSegStatus(seg2, SegmentStatusEnum.READY); cube = mgr.getCube(cube.getName()); assertEquals(2, cube.getSegments().size()); SegmentRange mergedSeg = cube.autoMergeCubeSegments(); assertTrue(mergedSeg != null); }
@Test public void testBuildCubeWithPartitionStartDate() throws IOException { Long PARTITION_DATE_START = 1513123200L; Long FIRST_BUILD_DATE_END = 1514764800L; Long SECOND_BUILD_DATE_END = 1540339200L; KylinConfig config = getTestConfig(); CubeManager cubeManager = CubeManager.getInstance(config); CubeInstance cube = cubeManager.getCube("test_kylin_cube_with_slr_empty"); cube.getDescriptor().setPartitionDateStart(PARTITION_DATE_START); CubeSegment segment = cubeManager.appendSegment(cube, new TSRange(0L, FIRST_BUILD_DATE_END), null, null, null); assertEquals(segment._getDateRangeStart(), PARTITION_DATE_START.longValue()); assertEquals(segment._getDateRangeEnd(), FIRST_BUILD_DATE_END.longValue()); cubeManager.updateCubeSegStatus(segment, SegmentStatusEnum.READY); segment = cubeManager.appendSegment(cube, new TSRange(0L, SECOND_BUILD_DATE_END), null, null, null); assertEquals(segment._getDateRangeStart(), FIRST_BUILD_DATE_END.longValue()); assertEquals(segment._getDateRangeEnd(), SECOND_BUILD_DATE_END.longValue()); }
@Test public void testSegmentOverlap() throws IOException { thrown.expect(RuntimeException.class); thrown.expectMessage("Segments has overlap"); HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv()); Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid")); HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" }); CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeInstance cube1 = cubeManager.getCube("ssb_cube1"); CubeInstance cube2 = cubeManager.getCube("ssb_cube2"); // 2012-01-01,2012-01-03 cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L)); // 2012-01-02,2012-01-04 cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325462400000L, 1325635200000L)); HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" }); }
@Test public void testDatePartition() { PartitionDesc partitionDesc = new PartitionDesc(); TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 1, "DATE_COLUMN", "string"); partitionDesc.setPartitionDateColumnRef(col); partitionDesc.setPartitionDateColumn(col.getCanonicalName()); partitionDesc.setPartitionDateFormat("yyyy-MM-dd"); TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22"), DateFormat.stringToMillis("2016-02-23")); String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals("UNKNOWN_ALIAS.DATE_COLUMN >= '2016-02-22' AND UNKNOWN_ALIAS.DATE_COLUMN < '2016-02-23'", condition); range = new TSRange(0L, 0L); condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals("1=0", condition); }
@Test public void testSegmentNotOverlap() throws IOException { HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv()); Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid")); HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" }); CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeInstance cube1 = cubeManager.getCube("ssb_cube1"); CubeInstance cube2 = cubeManager.getCube("ssb_cube2"); // 2012-01-01,2012-01-03 cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L)); // 2012-01-03,2012-01-04 cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325548800000L, 1325635200000L)); // not throw exception HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" }); }
@Test public void testTimePartition() { PartitionDesc partitionDesc = new PartitionDesc(); TblColRef col = TblColRef.mockup(TableDesc.mockup("DEFAULT.TABLE_NAME"), 2, "HOUR_COLUMN", "string"); partitionDesc.setPartitionTimeColumnRef(col); partitionDesc.setPartitionTimeColumn(col.getCanonicalName()); partitionDesc.setPartitionTimeFormat("HH"); TSRange range = new TSRange(DateFormat.stringToMillis("2016-02-22 00:00:00"), DateFormat.stringToMillis("2016-02-23 01:00:00")); String condition = partitionConditionBuilder.buildDateRangeCondition(partitionDesc, null, range); Assert.assertEquals("UNKNOWN_ALIAS.HOUR_COLUMN >= '00' AND UNKNOWN_ALIAS.HOUR_COLUMN < '01'", condition); }
@Override public TSRange getTSRange() { if (tsRange != null) return tsRange; tsRange = new TSRange(seg._getDateRangeStart(), seg._getDateRangeEnd()); return tsRange; }