int buildingSize = getBuildingSegments().size(); if (buildingSize > 0) { logger.debug("Cube " + cubeName + " has " + buildingSize + " building segments"); Segments<T> readySegs = getSegments(SegmentStatusEnum.READY); Segments mergingSegs = new Segments(); if (buildingSize > 0) { for (ISegment building : getBuildingSegments()) { mergingSegs.add(ready); removeLatestSegmentByVolatileRange(readySegs, volatileRange); readySegs.removeAll(mergingSegs); long toMergeRange = timeRanges[i]; for (int s = 0; s < readySegs.size(); s++) { ISegment seg = readySegs.get(s); TSRange tsRange = new TSRange(seg.getTSRange().start.v, seg.getTSRange().start.v + toMergeRange); Pair<T, T> p = readySegs.getSubList(s, readySegs.size()) // .findMergeOffsetsByDateRange(tsRange, toMergeRange); if (p != null && p.getSecond().getTSRange().end.v - p.getFirst().getTSRange().start.v >= toMergeRange) return new SegmentRange(p.getFirst().getSegRange().start.v, p.getSecond().getSegRange().end.v);
public Pair<Boolean, Boolean> fitInSegments(ISegment newOne) { if (this.isEmpty()) return null; ISegment first = this.get(0); ISegment last = this.get(this.size() - 1); Endpoint start = newOne.getSegRange().start; Endpoint end = newOne.getSegRange().end; boolean startFit = false; boolean endFit = false; for (ISegment sss : this) { if (sss == newOne) continue; startFit = startFit || (start.equals(sss.getSegRange().start) || start.equals(sss.getSegRange().end)); endFit = endFit || (end.equals(sss.getSegRange().start) || end.equals(sss.getSegRange().end)); } if (!startFit && endFit && newOne == first) startFit = true; if (!endFit && startFit && newOne == last) endFit = true; return Pair.newPair(startFit, endFit); }
if (cubeCopy.getSegments().isEmpty()) throw new IllegalArgumentException("Cube " + cubeCopy + " has no segments"); checkCubeIsPartitioned(cubeCopy); if (cubeCopy.getSegments().getFirstSegment().isOffsetCube()) { if (mergingSegments.size() <= 1) throw new IllegalArgumentException("Range " + newSegment.getSegRange() + " must contain at least 2 segments, but there is " + mergingSegments.size()); CubeSegment first = mergingSegments.get(0); CubeSegment last = mergingSegments.get(mergingSegments.size() - 1); if (force == false) { checkReadyForMerge(mergingSegments); newSegment.setTSRange(null); } else { newSegment.setTSRange(new TSRange(mergingSegments.getTSStart(), mergingSegments.getTSEnd())); newSegment.setSegRange(null);
if (this.isEmpty()) return; Segments<T> all = new Segments<T>(this); Collections.sort(all); boolean isOffsetCube = all.get(0).isOffsetCube(); for (ISegment seg : all) { seg.validate(); List<ISegment> ready = Lists.newArrayListWithCapacity(all.size()); List<ISegment> news = Lists.newArrayListWithCapacity(all.size()); for (ISegment seg : all) { if (seg.getStatus() == SegmentStatusEnum.READY) Pair<Boolean, Boolean> pair = all.fitInSegments(seg); boolean startFit = pair.getFirst(); boolean endFit = pair.getSecond();
Segments tobe = (Segments) this.clone(); if (newSegment != null && !tobe.contains(newSegment)) { tobe.add(newSegment); if (tobe.size() == 0) return tobe; ISegment firstSeg = tobe.getFirst(); firstSeg.validate(); for (int i = 0, j = 1; j < tobe.size();) { ISegment is = (ISegment) tobe.get(i); ISegment js = (ISegment) tobe.get(j); js.validate(); if (!isNew(is) && !isReady(is)) { tobe.remove(i); continue; if (!isNew(js) && !isReady(js)) { tobe.remove(j); continue; if (isReady(is) && isReady(js) || isNew(is) && isNew(js)) { tobe.remove(i); } else { tobe.remove(j);
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeManager = CubeManager.getInstance(context.getConfig()); final CubeInstance cubeCopy = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite(); final String segmentId = CubingExecutableUtil.getSegmentId(this.getParams()); final CubeSegment segCopy = cubeCopy.getSegmentById(segmentId); Preconditions.checkNotNull(segCopy, "Cube segment '" + segmentId + "' not found."); Segments<CubeSegment> mergingSegs = cubeCopy.getMergingSegments(segCopy); Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment not exist."); Collections.sort(mergingSegs); final CubeSegment first = mergingSegs.get(0); final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1); segCopy.setSegRange(new SegmentRange(first.getSegRange().start, last.getSegRange().end)); segCopy.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart()); segCopy.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd()); segCopy.setTSRange(new TSRange(mergingSegs.getTSStart(), mergingSegs.getTSEnd())); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToUpdateSegs(segCopy); try { cubeManager.updateCube(update); return ExecuteResult.createSucceed(); } catch (IOException e) { logger.error("fail to update cube segment offset", e); return ExecuteResult.createError(e); } }
public Pair<T, T> findMergeOffsetsByDateRange(TSRange tsRange, long skipSegDateRangeCap) { // must be offset cube Segments result = new Segments(); for (ISegment seg : this) { // include if date range overlaps if (tsRange.overlaps(seg.getTSRange())) { // reject too big segment if (seg.getTSRange().duration() > skipSegDateRangeCap) break; // reject holes if (result.size() > 0 && !result.getLast().getSegRange().connects(seg.getSegRange())) break; result.add(seg); } } if (result.size() <= 1) return null; else return (Pair<T, T>) Pair.newPair(result.getFirst(), result.getLast()); }
/** * when clear all segments, it's supposed to reinitialize the CubeInstance */ public CubeInstance clearSegments(CubeInstance cube) throws IOException { try (AutoLock lock = cubeMapLock.lockForWrite()) { cube = cube.latestCopyForWrite(); // get a latest copy CubeUpdate update = new CubeUpdate(cube); update.setToRemoveSegs(cube.getSegments().toArray(new CubeSegment[cube.getSegments().size()])); update.setCuboids(Maps.<Long, Long> newHashMap()); update.setCuboidsRecommend(Sets.<Long> newHashSet()); update.setUpdateTableSnapshotPath(Maps.<String, String> newHashMap()); update.setCreateTimeUTC(System.currentTimeMillis()); update.setCuboidLastOptimized(0L); return updateCube(update); } }
private void checkAllowParallelBuilding(CubeInstance cube) { if (cube.getConfig().isCubePlannerEnabled()) { if (cube.getCuboids() == null) { Segments<CubeSegment> cubeSegments = cube.getSegments(); if (cubeSegments.size() > 0 && cubeSegments.getSegments(SegmentStatusEnum.READY).size() <= 0) { throw new BadRequestException("The cube " + cube.getName() + " has segments " + cubeSegments + ", but none of them is READY. It's not allowed for parallel building"); } } } }
@Before public void setUp() throws Exception { this.createTestMetadata(); cube = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready"); cubeSegment = cube.getSegments().get(0); flatTableDesc = EngineFactory.getJoinedFlatTableDesc(cubeSegment); }
newCubeInstance.setDescName(newCubeDescName); newCubeInstance.updateRandomUuid(); Iterator<CubeSegment> segmentIterator = newCubeInstance.getSegments().iterator(); CubeSegment currentSeg = null; while (segmentIterator.hasNext()) { cubeInstance.setSegments(new Segments<CubeSegment>()); cubeInstance.setStatus(RealizationStatusEnum.DISABLED); store.checkAndPutResource(cubeInstance.getResourcePath(), cubeInstance, CubeManager.CUBE_SERIALIZER);
@Test public void copyCubeSegmentTest() { int origSegCount = cubeInstance.getSegments().size(); CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance); CubeSegment mockSeg = new CubeSegment(); mockSeg.setUuid(RandomUtil.randomUUID().toString()); mockSeg.setStorageLocationIdentifier(RandomUtil.randomUUID().toString()); mockSeg.setStatus(SegmentStatusEnum.READY); newCubeInstance.getSegments().add(mockSeg); Assert.assertEquals(origSegCount, cubeInstance.getSegments().size()); Assert.assertEquals(origSegCount + 1, newCubeInstance.getSegments().size()); }
public void removeLatestSegmentByVolatileRange(Segments<T> segs, long volatileRange) { if(volatileRange <= 0) { return; } long latestSegEndTs = Long.MIN_VALUE; for(T seg: segs) { latestSegEndTs = Math.max(latestSegEndTs, seg.getTSRange().end.v); } Segments volatileSegs = new Segments(); for(T seg: segs) { if(seg.getTSRange().end.v + volatileRange > latestSegEndTs) { logger.warn("segment in volatile range: seg:" + seg.toString() + "rangeStart:" + seg.getTSRange().start.v + ", rangeEnd" + seg.getTSRange().end.v); volatileSegs.add(seg); } } segs.removeAll(volatileSegs); }
private boolean validateCubeInstance(CubeInstance cubeInstance) { if (cubeInstance == null) { logger.error("This cube does not exist."); return false; } if (cubeInstance.getSegments().isEmpty()) { logger.error("No segments in this cube, no need to extend."); return false; } return true; }
public void setSegments(Segments segments) { this.segments = new Segments<>(segments); }