mergedSegment.setDimensionRangeInfoMap(mergedSegDimRangeMap); if (isOffsetCube) { SegmentRange.TSRange tsRange = new SegmentRange.TSRange(tsStartMin, tsEndMax); mergedSegment.setTSRange(tsRange);
mergedSegment.setDimensionRangeInfoMap(mergedSegDimRangeMap); if (isOffsetCube) { SegmentRange.TSRange tsRange = new SegmentRange.TSRange(tsStartMin, tsEndMax); mergedSegment.setTSRange(tsRange);
&& DateFormat.stringToMillis(maxValue) != timeMaxValue) { segment.setTSRange( new TSRange(DateFormat.stringToMillis(minValue), DateFormat.stringToMillis(maxValue) + 1));
hole.setName(CubeSegment.makeSegmentName(null, hole.getSegRange(), modelDesc)); } else { hole.setTSRange(new TSRange(first.getTSRange().end.v, second.getTSRange().start.v)); hole.setName(CubeSegment.makeSegmentName(hole.getTSRange(), null, modelDesc));
TSRange tsRange = new TSRange(seg.getTSRange().start.v, seg.getTSRange().start.v + toMergeRange);
CubeSegment appendSegment(CubeInstance cube, TSRange tsRange, SegmentRange segRange, Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd) throws IOException { CubeInstance cubeCopy = cube.latestCopyForWrite(); // get a latest copy checkInputRanges(tsRange, segRange); // fix start/end a bit if (cubeCopy.getModel().getPartitionDesc().isPartitioned()) { // if missing start, set it to where last time ends if (tsRange != null && tsRange.start.v == 0) { CubeDesc cubeDesc = cubeCopy.getDescriptor(); CubeSegment last = cubeCopy.getLastSegment(); if (last == null) tsRange = new TSRange(cubeDesc.getPartitionDateStart(), tsRange.end.v); else if (!last.isOffsetCube()) tsRange = new TSRange(last.getTSRange().end.v, tsRange.end.v); } } else { // full build tsRange = null; segRange = null; } CubeSegment newSegment = newSegment(cubeCopy, tsRange, segRange); newSegment.setSourcePartitionOffsetStart(sourcePartitionOffsetStart); newSegment.setSourcePartitionOffsetEnd(sourcePartitionOffsetEnd); validateNewSegments(cubeCopy, newSegment); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToAddSegs(newSegment); updateCube(update); return newSegment; }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeManager = CubeManager.getInstance(context.getConfig()); final CubeInstance cubeCopy = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite(); final String segmentId = CubingExecutableUtil.getSegmentId(this.getParams()); final CubeSegment segCopy = cubeCopy.getSegmentById(segmentId); Preconditions.checkNotNull(segCopy, "Cube segment '" + segmentId + "' not found."); Segments<CubeSegment> mergingSegs = cubeCopy.getMergingSegments(segCopy); Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment not exist."); Collections.sort(mergingSegs); final CubeSegment first = mergingSegs.get(0); final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1); segCopy.setSegRange(new SegmentRange(first.getSegRange().start, last.getSegRange().end)); segCopy.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart()); segCopy.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd()); segCopy.setTSRange(new TSRange(mergingSegs.getTSStart(), mergingSegs.getTSEnd())); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToUpdateSegs(segCopy); try { cubeManager.updateCube(update); return ExecuteResult.createSucceed(); } catch (IOException e) { logger.error("fail to update cube segment offset", e); return ExecuteResult.createError(e); } }
private CubeSegment newSegment(CubeInstance cube, TSRange tsRange, SegmentRange segRange) { DataModelDesc modelDesc = cube.getModel(); CubeSegment segment = new CubeSegment(); segment.setUuid(RandomUtil.randomUUID().toString()); segment.setName(CubeSegment.makeSegmentName(tsRange, segRange, modelDesc)); segment.setCreateTimeUTC(System.currentTimeMillis()); segment.setCubeInstance(cube); // let full build range be backward compatible if (tsRange == null && segRange == null) tsRange = new TSRange(0L, Long.MAX_VALUE); segment.setTSRange(tsRange); segment.setSegRange(segRange); segment.setStatus(SegmentStatusEnum.NEW); segment.setStorageLocationIdentifier(generateStorageLocation()); segment.setCubeInstance(cube); segment.validate(); return segment; }
public Pair<T, T> findMergeOffsetsByDateRange(TSRange tsRange, long skipSegDateRangeCap) { // must be offset cube Segments result = new Segments(); for (ISegment seg : this) { // include if date range overlaps if (tsRange.overlaps(seg.getTSRange())) { // reject too big segment if (seg.getTSRange().duration() > skipSegDateRangeCap) break; // reject holes if (result.size() > 0 && !result.getLast().getSegRange().connects(seg.getSegRange())) break; result.add(seg); } } if (result.size() <= 1) return null; else return (Pair<T, T>) Pair.newPair(result.getFirst(), result.getLast()); }
@Override public TSRange getTSRange() { if (tsRange != null) return tsRange; tsRange = new TSRange(seg._getDateRangeStart(), seg._getDateRangeEnd()); return tsRange; }
public TSRange(Long start, Long end) { // [0, Long.MAX_VALUE) is full build (for historic reason) super(new Endpoint(isInfinite(start, end) ? 0 : start, isInfinite(start, end), false), // new Endpoint(isInfinite(start, end) ? Long.MAX_VALUE : end, false, isInfinite(start, end))); }
private void run(String cubeName, long startDate, long endDate, String buildType) throws IOException, JobException { CubeInstance cube = cubeManager.getCube(cubeName); Preconditions.checkArgument(cube != null, "Cube named " + cubeName + " does not exist!!!"); CubeBuildTypeEnum buildTypeEnum = CubeBuildTypeEnum.valueOf(buildType); Preconditions.checkArgument(buildTypeEnum != null, "Build type named " + buildType + " does not exist!!!"); submitJob(cube, new TSRange(startDate, endDate), buildTypeEnum, false, "SYSTEM"); }
CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L)); mgr.updateCubeSegStatus(seg1, SegmentStatusEnum.READY); CubeSegment seg3 = mgr.appendSegment(cube, new TSRange(2000L, 4000L)); mgr.updateCubeSegStatus(seg3, SegmentStatusEnum.READY); CubeSegment seg4 = mgr.appendSegment(cube, new TSRange(4000L, 8000L)); mgr.updateCubeSegStatus(seg4, SegmentStatusEnum.READY); CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 2000L)); mgr.updateCubeSegStatus(seg2, SegmentStatusEnum.READY);
&& DateFormat.stringToMillis(maxValue) != timeMaxValue) { segment.setTSRange( new TSRange(DateFormat.stringToMillis(minValue), DateFormat.stringToMillis(maxValue) + 1));
CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L)); cube = readySegment(cube, seg1); CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 2000L)); cube = readySegment(cube, seg2); assertEquals(new TSRange(1000L, 2000L), seg2.getTSRange()); assertEquals(new TSRange(1000L, 2000L), seg2.getSegRange()); CubeSegment merge = mgr.mergeSegments(cube, new TSRange(0L, 2000L), null, true); assertEquals(2, cube.getSegments().size()); // older version of cube assertEquals(new TSRange(0L, 2000L), merge.getTSRange()); assertEquals(new TSRange(0L, 2000L), merge.getSegRange()); assertEquals(SegmentStatusEnum.NEW, merge.getStatus()); mgr.mergeSegments(cube, new TSRange(500L, 2500L), null, true); fail(); } catch (IllegalArgumentException ex) { CubeSegment merge2 = mgr.mergeSegments(cube, new TSRange(0L, 2500L), null, true); assertEquals(new TSRange(0L, 2000L), merge2.getTSRange()); assertEquals(new TSRange(0L, 2000L), merge2.getSegRange());
TSRange tsRange = new TSRange(seg.getTSRange().start.v, seg.getTSRange().start.v + toMergeRange);
CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L)); mgr.updateCubeSegStatus(seg1, SegmentStatusEnum.READY); CubeSegment seg3 = mgr.appendSegment(cube, new TSRange(2000L, 4000L)); mgr.updateCubeSegStatus(seg3, SegmentStatusEnum.READY); CubeSegment seg4 = mgr.appendSegment(cube, new TSRange(4000L, 8000L)); mgr.updateCubeSegStatus(seg4, SegmentStatusEnum.READY); CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 2000L)); mgr.updateCubeSegStatus(seg2, SegmentStatusEnum.READY);
hole.setName(CubeSegment.makeSegmentName(null, hole.getSegRange(), modelDesc)); } else { hole.setTSRange(new TSRange(first.getTSRange().end.v, second.getTSRange().start.v)); hole.setName(CubeSegment.makeSegmentName(hole.getTSRange(), null, modelDesc));