public Set<String> toSet(Set<String> partColsQueried) { Set<String> ret = Sets.newHashSet(); for (Map.Entry<String, RangesPartitionTimeline> entry : entrySet()) { if (partColsQueried.contains(entry.getKey())) { ret.add(entry.getKey() + ":" + entry.getValue().getRanges()); } } return ret; }
/** * Returns time covered by candidate for each part column. * * @param cand * @return */ private Map<String, Long> getTimeCoveredForEachPartCol(Candidate cand) { Map<String, Long> ret = Maps.newHashMap(); PartitionRangesForPartitionColumns partitionRangesForPartitionColumns = new PartitionRangesForPartitionColumns(); for (FactPartition part : cand.getParticipatingPartitions()) { if (part.isFound()) { try { partitionRangesForPartitionColumns.add(part); } catch (LensException e) { log.error("invalid partition: ", e); } } } for (Map.Entry<String, RangesPartitionTimeline> entry : partitionRangesForPartitionColumns.entrySet()) { ret.put(entry.getKey(), entry.getValue().getTimeCovered()); } return ret; }
@Override public boolean initFromProperties(Map<String, String> properties) throws LensException { ranges.clear(); String rangesStr = MetastoreUtil.getNamedStringValue(properties, "ranges"); if (!Strings.isNullOrEmpty(rangesStr)) { String[] split = rangesStr.split("\\s*,\\s*"); if (split.length % 2 == 1) { throw new LensException("Ranges incomplete"); } for (int i = 0; i < split.length; i += 2) { ranges.add(TimePartitionRange.parseFrom(getUpdatePeriod(), split[i], split[i + 1])); } } return isConsistent(); }
public void add(String partCol, TimePartition partition) throws LensException { if (get(partCol) == null) { put(partCol, new RangesPartitionTimeline("", UpdatePeriod.values()[0], partCol)); } get(partCol).add(partition.withUpdatePeriod(UpdatePeriod.values()[0]) .rangeUpto(partition.next().withUpdatePeriod(UpdatePeriod.values()[0]))); }
@Override public boolean add(TimePartitionRange partitionRange) throws LensException { // Adding partition range to the timeline. Will have to find if any of the sub ranges // intersects. If yes, add only remaining sub ranges, else add the given range as a new sub range. int strictlyAfterIndex = getStrictlyAfterIndex(partitionRange.getBegin()); while (strictlyAfterIndex < ranges.size() && partitionRange.isValidAndNonEmpty()) { if (partitionRange.getEnd().before(ranges.get(strictlyAfterIndex).getBegin())) { // partition begin and end both are strictly before ranges[strictlyAfterIndex]. Add as new sub range. ranges.add(strictlyAfterIndex, partitionRange); partitionRange = partitionRange.getEnd().emptyRange(); break; } else { // begin is before ranges[strictlyAfterIndex], end is not. // extend ranges[strictlyAfterIndex] and add remaining range, if any. ranges.get(strictlyAfterIndex).setBegin(partitionRange.getBegin()); if (ranges.get(strictlyAfterIndex).getEnd().before(partitionRange.getEnd())) { partitionRange = ranges.get(strictlyAfterIndex).getEnd().rangeUpto(partitionRange.getEnd()); } else { // No remaining range, end was before ranges[strictlyAfterIndex].end partitionRange = ranges.get(strictlyAfterIndex).getEnd().emptyRange(); } strictlyAfterIndex++; } } if (strictlyAfterIndex == ranges.size() && partitionRange.isValidAndNonEmpty()) { ranges.add(partitionRange); } mergeRanges(); return true; }
@Override public boolean exists(TimePartition toCheck) { if (isEmpty()) { return false; } for (TimePartitionRange range : ranges) { if (range.contains(toCheck)) { return true; } } return false; }
@Override public boolean drop(TimePartition toDrop) throws LensException { int ind = getStrictlyAfterIndex(toDrop); if (ind == 0) { return true; // nothing to do } if (ranges.get(ind - 1).getBegin().equals(toDrop)) { ranges.get(ind - 1).setBegin(toDrop.next()); } else if (ranges.get(ind - 1).getEnd().previous().equals(toDrop)) { ranges.get(ind - 1).setEnd(toDrop); } else { TimePartition end = ranges.get(ind - 1).getEnd(); ranges.get(ind - 1).setEnd(toDrop); ranges.add(ind, toDrop.next().rangeUpto(end)); } if (ranges.get(ind - 1).isEmpty()) { ranges.remove(ind - 1); } return true; }
public void add(String partCol, TimePartition partition) throws LensException { if (get(partCol) == null) { put(partCol, new RangesPartitionTimeline("", UpdatePeriod.values()[0], partCol)); } get(partCol).add(partition.withUpdatePeriod(UpdatePeriod.values()[0]) .rangeUpto(partition.next().withUpdatePeriod(UpdatePeriod.values()[0]))); }
@Override public boolean add(TimePartitionRange partitionRange) throws LensException { // Adding partition range to the timeline. Will have to find if any of the sub ranges // intersects. If yes, add only remaining sub ranges, else add the given range as a new sub range. int strictlyAfterIndex = getStrictlyAfterIndex(partitionRange.getBegin()); while (strictlyAfterIndex < ranges.size() && partitionRange.isValidAndNonEmpty()) { if (partitionRange.getEnd().before(ranges.get(strictlyAfterIndex).getBegin())) { // partition begin and end both are strictly before ranges[strictlyAfterIndex]. Add as new sub range. ranges.add(strictlyAfterIndex, partitionRange); partitionRange = partitionRange.getEnd().emptyRange(); break; } else { // begin is before ranges[strictlyAfterIndex], end is not. // extend ranges[strictlyAfterIndex] and add remaining range, if any. ranges.get(strictlyAfterIndex).setBegin(partitionRange.getBegin()); if (ranges.get(strictlyAfterIndex).getEnd().before(partitionRange.getEnd())) { partitionRange = ranges.get(strictlyAfterIndex).getEnd().rangeUpto(partitionRange.getEnd()); } else { // No remaining range, end was before ranges[strictlyAfterIndex].end partitionRange = ranges.get(strictlyAfterIndex).getEnd().emptyRange(); } strictlyAfterIndex++; } } if (strictlyAfterIndex == ranges.size() && partitionRange.isValidAndNonEmpty()) { ranges.add(partitionRange); } mergeRanges(); return true; }
@Override public boolean exists(TimePartition toCheck) { if (isEmpty()) { return false; } for (TimePartitionRange range : ranges) { if (range.contains(toCheck)) { return true; } } return false; }
@Override public boolean drop(TimePartition toDrop) throws LensException { int ind = getStrictlyAfterIndex(toDrop); if (ind == 0) { return true; // nothing to do } if (ranges.get(ind - 1).getBegin().equals(toDrop)) { ranges.get(ind - 1).setBegin(toDrop.next()); } else if (ranges.get(ind - 1).getEnd().previous().equals(toDrop)) { ranges.get(ind - 1).setEnd(toDrop); } else { TimePartition end = ranges.get(ind - 1).getEnd(); ranges.get(ind - 1).setEnd(toDrop); ranges.add(ind, toDrop.next().rangeUpto(end)); } if (ranges.get(ind - 1).isEmpty()) { ranges.remove(ind - 1); } return true; }
@Override public boolean initFromProperties(Map<String, String> properties) throws LensException { ranges.clear(); String rangesStr = MetastoreUtil.getNamedStringValue(properties, "ranges"); if (!Strings.isNullOrEmpty(rangesStr)) { String[] split = rangesStr.split("\\s*,\\s*"); if (split.length % 2 == 1) { throw new LensException("Ranges incomplete"); } for (int i = 0; i < split.length; i += 2) { ranges.add(TimePartitionRange.parseFrom(getUpdatePeriod(), split[i], split[i + 1])); } } return isConsistent(); }
@Override public TimePartition latest() { if (isEmpty()) { return null; } return ranges.get(ranges.size() - 1).getEnd().previous(); }
public Set<String> toSet(Set<String> partColsQueried) { Set<String> ret = Sets.newHashSet(); for (Map.Entry<String, RangesPartitionTimeline> entry : entrySet()) { if (partColsQueried.contains(entry.getKey())) { ret.add(entry.getKey() + ":" + entry.getValue().getRanges()); } } return ret; }
@Override public boolean add(TimePartition partition) throws LensException { int ind = getStrictlyAfterIndex(partition); int added = 0; if (ind > 0) {
/** * Returns time covered by candidate for each part column. * * @param cand * @return */ private Map<String, Long> getTimeCoveredForEachPartCol(Candidate cand) { Map<String, Long> ret = Maps.newHashMap(); PartitionRangesForPartitionColumns partitionRangesForPartitionColumns = new PartitionRangesForPartitionColumns(); for (FactPartition part : cand.getParticipatingPartitions()) { if (part.isFound()) { try { partitionRangesForPartitionColumns.add(part); } catch (LensException e) { log.error("invalid partition: ", e); } } } for (Map.Entry<String, RangesPartitionTimeline> entry : partitionRangesForPartitionColumns.entrySet()) { ret.put(entry.getKey(), entry.getValue().getTimeCovered()); } return ret; }
@Override public TimePartition latest() { if (isEmpty()) { return null; } return ranges.get(ranges.size() - 1).getEnd().previous(); }
private int getStrictlyAfterIndex(TimePartition part) { int start = 0; int end = getRanges().size(); int mid; while (end - start > 0) { mid = (start + end) / 2; if (ranges.get(mid).getBegin().after(part)) { end = mid; } else { start = mid + 1; } } return end; }
@Override public boolean add(TimePartition partition) throws LensException { int ind = getStrictlyAfterIndex(partition); int added = 0; if (ind > 0) {
@Override public boolean isConsistent() { if (isEmpty()) { return true; } if (!ranges.get(0).getBegin().before(ranges.get(0).getEnd())) { return false; } for (int i = 0; i < ranges.size() - 1; i++) { if (!ranges.get(i).getEnd().before(ranges.get(i + 1).getBegin())) { return false; } if (!ranges.get(i + 1).getBegin().before(ranges.get(i + 1).getEnd())) { return false; } } return true; }