@Override public boolean exists(TimePartition toCheck) { if (isEmpty()) { return false; } for (TimePartitionRange range : ranges) { if (range.contains(toCheck)) { return true; } } return false; }
public TimePartitionRange rangeFrom(TimePartition from) throws LensException { return new TimePartitionRange(from, this); }
private void mergeRanges() { for (int i = 0; i < ranges.size() - 1; i++) { if (ranges.get(i).getEnd().compareTo(ranges.get(i + 1).getBegin()) >= 0) { TimePartitionRange removed = ranges.remove(i + 1); ranges.get(i).setEnd(TimePartition.max(removed.getEnd(), ranges.get(i).getEnd())); i--; // check again at same index } else if (ranges.get(i).isEmpty()) { ranges.remove(i); i--; } } }
int added = 0; if (ind > 0) { if (ranges.get(ind - 1).contains(partition)) { return true; if (ranges.get(ind - 1).getEnd().equals(partition)) { added++; ranges.get(ind - 1).setEnd(partition.next()); if (partition.equals(ranges.get(ind).getBegin().previous())) { added++; ranges.get(ind).setBegin(partition); break; case 2: ranges.get(ind - 1).setEnd(ranges.get(ind).getEnd()); ranges.remove(ind); break;
assertEquals(range, tenLater.rangeFrom(nowPartition)); assertEquals(range.size(), 10); assertEquals(nowPartition.singletonRange().size(), 1); assertTrue(range.contains(nowPartition)); assertFalse(range.contains(tenLater)); assertEquals(TimePartitionRange.parseFrom(up, nowStr, tenLaterStr), range); assertEquals(TimePartitionRange.parseFrom(up, "[" + nowStr, tenLaterStr + ")"), range); assertEquals(TimePartitionRange.parseFrom(up, "[" + nowStr, tenLaterStr + "]"), nowPartition.rangeUpto(tenLater.next())); assertEquals(TimePartitionRange.parseFrom(up, "(" + nowStr, tenLaterStr + "]"), nowPartition.next().rangeUpto( tenLater.next())); assertEquals(TimePartitionRange.parseFrom(up, "(" + nowStr, tenLaterStr + ")"), nowPartition.next().rangeUpto(tenLater));
@Override public boolean add(TimePartitionRange partitionRange) throws LensException { // Adding partition range to the timeline. Will have to find if any of the sub ranges // intersects. If yes, add only remaining sub ranges, else add the given range as a new sub range. int strictlyAfterIndex = getStrictlyAfterIndex(partitionRange.getBegin()); while (strictlyAfterIndex < ranges.size() && partitionRange.isValidAndNonEmpty()) { if (partitionRange.getEnd().before(ranges.get(strictlyAfterIndex).getBegin())) { // partition begin and end both are strictly before ranges[strictlyAfterIndex]. Add as new sub range. ranges.add(strictlyAfterIndex, partitionRange); partitionRange = partitionRange.getEnd().emptyRange(); break; } else { // begin is before ranges[strictlyAfterIndex], end is not. // extend ranges[strictlyAfterIndex] and add remaining range, if any. ranges.get(strictlyAfterIndex).setBegin(partitionRange.getBegin()); if (ranges.get(strictlyAfterIndex).getEnd().before(partitionRange.getEnd())) { partitionRange = ranges.get(strictlyAfterIndex).getEnd().rangeUpto(partitionRange.getEnd()); } else { // No remaining range, end was before ranges[strictlyAfterIndex].end partitionRange = ranges.get(strictlyAfterIndex).getEnd().emptyRange(); } strictlyAfterIndex++; } } if (strictlyAfterIndex == ranges.size() && partitionRange.isValidAndNonEmpty()) { ranges.add(partitionRange); } mergeRanges(); return true; }
@Override public boolean initFromProperties(Map<String, String> properties) throws LensException { ranges.clear(); String rangesStr = MetastoreUtil.getNamedStringValue(properties, "ranges"); if (!Strings.isNullOrEmpty(rangesStr)) { String[] split = rangesStr.split("\\s*,\\s*"); if (split.length % 2 == 1) { throw new LensException("Ranges incomplete"); } for (int i = 0; i < split.length; i += 2) { ranges.add(TimePartitionRange.parseFrom(getUpdatePeriod(), split[i], split[i + 1])); } } return isConsistent(); }
@Override public TimePartition latest() { if (isEmpty()) { return null; } return ranges.get(ranges.size() - 1).getEnd().previous(); }
@Override public boolean hasNext() { if (cur == null || !cur.hasNext()) { if (!uber.hasNext()) { return false; } cur = uber.next().iterator(); } return cur.hasNext(); }
private int getStrictlyAfterIndex(TimePartition part) { int start = 0; int end = getRanges().size(); int mid; while (end - start > 0) { mid = (start + end) / 2; if (ranges.get(mid).getBegin().after(part)) { end = mid; } else { start = mid + 1; } } return end; }
public static void addParts(Collection<String> partitions, UpdatePeriod updatePeriod, Date from, Date to) { try { for (TimePartition timePartition : TimePartitionRange.between(from, to, updatePeriod)) { partitions.add(timePartition.toString()); } } catch (LensException e) { throw new IllegalArgumentException(e); } }
private void mergeRanges() { for (int i = 0; i < ranges.size() - 1; i++) { if (ranges.get(i).getEnd().compareTo(ranges.get(i + 1).getBegin()) >= 0) { TimePartitionRange removed = ranges.remove(i + 1); ranges.get(i).setEnd(TimePartition.max(removed.getEnd(), ranges.get(i).getEnd())); i--; // check again at same index } else if (ranges.get(i).isEmpty()) { ranges.remove(i); i--; } } }
int added = 0; if (ind > 0) { if (ranges.get(ind - 1).contains(partition)) { return true; if (ranges.get(ind - 1).getEnd().equals(partition)) { added++; ranges.get(ind - 1).setEnd(partition.next()); if (partition.equals(ranges.get(ind).getBegin().previous())) { added++; ranges.get(ind).setBegin(partition); break; case 2: ranges.get(ind - 1).setEnd(ranges.get(ind).getEnd()); ranges.remove(ind); break;
@Override public boolean add(TimePartitionRange partitionRange) throws LensException { // Adding partition range to the timeline. Will have to find if any of the sub ranges // intersects. If yes, add only remaining sub ranges, else add the given range as a new sub range. int strictlyAfterIndex = getStrictlyAfterIndex(partitionRange.getBegin()); while (strictlyAfterIndex < ranges.size() && partitionRange.isValidAndNonEmpty()) { if (partitionRange.getEnd().before(ranges.get(strictlyAfterIndex).getBegin())) { // partition begin and end both are strictly before ranges[strictlyAfterIndex]. Add as new sub range. ranges.add(strictlyAfterIndex, partitionRange); partitionRange = partitionRange.getEnd().emptyRange(); break; } else { // begin is before ranges[strictlyAfterIndex], end is not. // extend ranges[strictlyAfterIndex] and add remaining range, if any. ranges.get(strictlyAfterIndex).setBegin(partitionRange.getBegin()); if (ranges.get(strictlyAfterIndex).getEnd().before(partitionRange.getEnd())) { partitionRange = ranges.get(strictlyAfterIndex).getEnd().rangeUpto(partitionRange.getEnd()); } else { // No remaining range, end was before ranges[strictlyAfterIndex].end partitionRange = ranges.get(strictlyAfterIndex).getEnd().emptyRange(); } strictlyAfterIndex++; } } if (strictlyAfterIndex == ranges.size() && partitionRange.isValidAndNonEmpty()) { ranges.add(partitionRange); } mergeRanges(); return true; }
@Override public boolean initFromProperties(Map<String, String> properties) throws LensException { ranges.clear(); String rangesStr = MetastoreUtil.getNamedStringValue(properties, "ranges"); if (!Strings.isNullOrEmpty(rangesStr)) { String[] split = rangesStr.split("\\s*,\\s*"); if (split.length % 2 == 1) { throw new LensException("Ranges incomplete"); } for (int i = 0; i < split.length; i += 2) { ranges.add(TimePartitionRange.parseFrom(getUpdatePeriod(), split[i], split[i + 1])); } } return isConsistent(); }
@Override public TimePartition latest() { if (isEmpty()) { return null; } return ranges.get(ranges.size() - 1).getEnd().previous(); }
@Override public boolean hasNext() { if (cur == null || !cur.hasNext()) { if (!uber.hasNext()) { return false; } cur = uber.next().iterator(); } return cur.hasNext(); }
private int getStrictlyAfterIndex(TimePartition part) { int start = 0; int end = getRanges().size(); int mid; while (end - start > 0) { mid = (start + end) / 2; if (ranges.get(mid).getBegin().after(part)) { end = mid; } else { start = mid + 1; } } return end; }
@Override public boolean drop(TimePartition toDrop) throws LensException { int ind = getStrictlyAfterIndex(toDrop); if (ind == 0) { return true; // nothing to do } if (ranges.get(ind - 1).getBegin().equals(toDrop)) { ranges.get(ind - 1).setBegin(toDrop.next()); } else if (ranges.get(ind - 1).getEnd().previous().equals(toDrop)) { ranges.get(ind - 1).setEnd(toDrop); } else { TimePartition end = ranges.get(ind - 1).getEnd(); ranges.get(ind - 1).setEnd(toDrop); ranges.add(ind, toDrop.next().rangeUpto(end)); } if (ranges.get(ind - 1).isEmpty()) { ranges.remove(ind - 1); } return true; }