public T getFirstSegment() { if (this == null || this.size() == 0) { return null; } else { return this.get(0); } }
private T getFirst() { assert this.size() != 0; return this.get(0); }
private void checkAllowParallelBuilding(CubeInstance cube) { if (cube.getConfig().isCubePlannerEnabled()) { if (cube.getCuboids() == null) { Segments<CubeSegment> cubeSegments = cube.getSegments(); if (cubeSegments.size() > 0 && cubeSegments.getSegments(SegmentStatusEnum.READY).size() <= 0) { throw new BadRequestException("The cube " + cube.getName() + " has segments " + cubeSegments + ", but none of them is READY. It's not allowed for parallel building"); } } } }
private void processToUpdateSegments(CubeUpdate update, Segments<CubeSegment> newSegs) { for (CubeSegment segment : update.getToUpdateSegs()) { for (int i = 0; i < newSegs.size(); i++) { if (newSegs.get(i).getUuid().equals(segment.getUuid())) { newSegs.set(i, segment); break; } } } }
public T getLatestBuiltSegment() { T latest = null; for (int i = this.size() - 1; i >= 0; i--) { T seg = this.get(i); if (seg.getLastBuildTime() > 0) { if (latest == null || seg.getLastBuildTime() > latest.getLastBuildTime()) latest = seg; } } return latest; }
public T getLatestReadySegment() { T latest = null; for (int i = this.size() - 1; i >= 0; i--) { T seg = this.get(i); if (seg.getStatus() != SegmentStatusEnum.READY) continue; if (latest == null || latest.getTSRange().end.v < seg.getTSRange().end.v) { latest = seg; } } return latest; }
private void checkReadyForMerge(Segments<CubeSegment> mergingSegments) { // check if the segments to be merged are continuous for (int i = 0; i < mergingSegments.size() - 1; i++) { if (!mergingSegments.get(i).getSegRange().connects(mergingSegments.get(i + 1).getSegRange())) throw new IllegalStateException("Merging segments must not have gaps between " + mergingSegments.get(i) + " and " + mergingSegments.get(i + 1)); } // check if the segments to be merged are not empty List<String> emptySegment = Lists.newArrayList(); for (CubeSegment seg : mergingSegments) { if (seg.getSizeKB() == 0 && seg.getInputRecords() == 0) { emptySegment.add(seg.getName()); } } if (emptySegment.size() > 0) { throw new IllegalArgumentException( "Empty cube segment found, couldn't merge unless 'forceMergeEmptySegment' set to true: " + emptySegment); } }
private void checkAllowBuilding(CubeInstance cube) { if (cube.getConfig().isCubePlannerEnabled()) { Segments<CubeSegment> readyPendingSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING); if (readyPendingSegments.size() > 0) { throw new BadRequestException("The cube " + cube.getName() + " has READY_PENDING segments " + readyPendingSegments + ". It's not allowed for building"); } } }
public static boolean isAbleToOptimizeCubingPlan(CubeSegment segment) { CubeInstance cube = segment.getCubeInstance(); if (!cube.getConfig().isCubePlannerEnabled()) return false; if (cube.getSegments(SegmentStatusEnum.READY_PENDING).size() > 0) { logger.info("Has read pending segments and will not enable cube planner."); return false; } List<CubeSegment> readySegments = cube.getSegments(SegmentStatusEnum.READY); List<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.NEW); if (newSegments.size() <= 1 && // (readySegments.size() == 0 || // (cube.getConfig().isCubePlannerEnabledForExistingCube() && readySegments.size() == 1 && readySegments.get(0).getSegRange().equals(segment.getSegRange())))) { return true; } else { return false; } } }
/** * when clear all segments, it's supposed to reinitialize the CubeInstance */ public CubeInstance clearSegments(CubeInstance cube) throws IOException { try (AutoLock lock = cubeMapLock.lockForWrite()) { cube = cube.latestCopyForWrite(); // get a latest copy CubeUpdate update = new CubeUpdate(cube); update.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()])); update.setCuboids(Maps.<Long, Long> newHashMap()); update.setCuboidsRecommend(Sets.<Long> newHashSet()); update.setUpdateTableSnapshotPath(Maps.<String, String> newHashMap()); update.setCreateTimeUTC(System.currentTimeMillis()); update.setCuboidLastOptimized(0L); return updateCube(update); } }
private void checkAllowOptimization(CubeInstance cube, Set<Long> cuboidsRecommend) { Segments<CubeSegment> buildingSegments = cube.getBuildingSegments(); if (buildingSegments.size() > 0) { throw new BadRequestException("The cube " + cube.getName() + " has building segments " + buildingSegments + ". It's not allowed for optimization"); } long baseCuboid = cube.getCuboidScheduler().getBaseCuboidId(); if (!cuboidsRecommend.contains(baseCuboid)) { throw new BadRequestException("The recommend cuboids should contain the base cuboid " + baseCuboid); } Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds(); if (currentCuboidSet.equals(cuboidsRecommend)) { throw new BadRequestException( "The recommend cuboids are the same as the current cuboids. It's no need to do optimization."); } }
public void checkEnableCubeCondition(CubeInstance cube) { aclEvaluate.checkProjectWritePermission(cube); Message msg = MsgPicker.getMsg(); String cubeName = cube.getName(); RealizationStatusEnum ostatus = cube.getStatus(); if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) { throw new BadRequestException( String.format(Locale.ROOT, msg.getENABLE_NOT_DISABLED_CUBE(), cubeName, ostatus)); } if (cube.getSegments(SegmentStatusEnum.READY).size() == 0) { throw new BadRequestException(String.format(Locale.ROOT, msg.getNO_READY_SEGMENT(), cubeName)); } if (!cube.getDescriptor().checkSignature()) { throw new BadRequestException( String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor())); } }
public Pair<T, T> findMergeOffsetsByDateRange(TSRange tsRange, long skipSegDateRangeCap) { // must be offset cube Segments result = new Segments(); for (ISegment seg : this) { // include if date range overlaps if (tsRange.overlaps(seg.getTSRange())) { // reject too big segment if (seg.getTSRange().duration() > skipSegDateRangeCap) break; // reject holes if (result.size() > 0 && !result.getLast().getSegRange().connects(seg.getSegRange())) break; result.add(seg); } } if (result.size() <= 1) return null; else return (Pair<T, T>) Pair.newPair(result.getFirst(), result.getLast()); }
@Test public void copyCubeSegmentTest() { int origSegCount = cubeInstance.getSegments().size(); CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance); CubeSegment mockSeg = new CubeSegment(); mockSeg.setUuid(RandomUtil.randomUUID().toString()); mockSeg.setStorageLocationIdentifier(RandomUtil.randomUUID().toString()); mockSeg.setStatus(SegmentStatusEnum.READY); newCubeInstance.getSegments().add(mockSeg); Assert.assertEquals(origSegCount, cubeInstance.getSegments().size()); Assert.assertEquals(origSegCount + 1, newCubeInstance.getSegments().size()); }
public CubeDesc updateCube(CubeInstance cube, CubeDesc desc, ProjectInstance project) throws IOException { aclEvaluate.checkProjectWritePermission(cube); Message msg = MsgPicker.getMsg(); String projectName = project.getName(); desc.setDraft(false); try { if (cube.getSegments().size() != 0 && !cube.getDescriptor().consistentWith(desc)) { throw new BadRequestException( String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC(), desc.getName())); } desc = updateCubeAndDesc(cube, desc, projectName, true); } catch (AccessDeniedException accessDeniedException) { throw new ForbiddenException(msg.getUPDATE_CUBE_NO_RIGHT()); } if (desc.isBroken()) { throw new BadRequestException(desc.getErrorsAsString()); } return desc; }
private CubeInstance cloneCubeInstance(CubeManager cubeManager, CubeInstance cube, String name) throws IOException { CubeInstance cubeClone = cubeManager.createCube(name, projectName, cube.getDescriptor(), cube.getOwner()); CubeUpdate cubeUpdate = new CubeUpdate(cubeClone.latestCopyForWrite()); cubeUpdate.setToAddSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()])); cubeUpdate.setStatus(RealizationStatusEnum.READY); return cubeManager.updateCube(cubeUpdate); } }
@Test public void testAutoMergeNormal() throws Exception { CubeManager mgr = CubeManager.getInstance(getTestConfig()); CubeInstance cube = mgr.getCube("test_kylin_cube_with_slr_empty").latestCopyForWrite(); cube.getDescriptor().setAutoMergeTimeRanges(new long[] { 2000, 6000 }); mgr.updateCube(new CubeUpdate(cube)); assertTrue(cube.needAutoMerge()); // no segment at first assertEquals(0, cube.getSegments().size()); // append first CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L), null, null, null); mgr.updateCubeSegStatus(seg1, SegmentStatusEnum.READY); CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 2000L), null, null, null); mgr.updateCubeSegStatus(seg2, SegmentStatusEnum.READY); cube = mgr.getCube(cube.getName()); assertEquals(2, cube.getSegments().size()); SegmentRange mergedSeg = cube.autoMergeCubeSegments(); assertTrue(mergedSeg != null); }
@Test public void testReloadCache() throws Exception { ResourceStore store = getStore(); // clean legacy in case last run failed store.deleteResource("/cube/a_whole_new_cube.json"); CubeDescManager cubeDescMgr = getCubeDescManager(); CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc"); cubeManager.createCube("a_whole_new_cube", "default", desc, null); CubeInstance createdCube = cubeManager.getCube("a_whole_new_cube"); assertEquals(0, createdCube.getSegments().size()); assertEquals(RealizationStatusEnum.DISABLED, createdCube.getStatus()); cubeManager.updateCubeStatus(createdCube, RealizationStatusEnum.READY); assertEquals(RealizationStatusEnum.READY, cubeManager.getCube("a_whole_new_cube").getStatus()); }