@Override public String toString() { return "[" + begin.getDateString() + ", " + end.getDateString() + ")"; }
public FactPartition(String partCol, TimePartition timePartition, FactPartition containingPart, Set<String> storageTables) { this(partCol, timePartition.getDate(), timePartition.getUpdatePeriod(), containingPart, null, storageTables); }
@Test(dataProvider = "update-periods") public void test(UpdatePeriod up) throws LensException { // Normal date object parsable String nowStr = up.format(NOW); // Create partition by date object or it's string representation -- both should be same. TimePartition nowPartition = TimePartition.of(up, NOW); TimePartition nowStrPartition = TimePartition.of(up, nowStr); assertEquals(nowPartition, nowStrPartition); // Test next and previous assertTrue(nowPartition.next().after(nowPartition)); assertTrue(nowPartition.previous().before(nowPartition)); // date parse failures should give lens exception assertEquals(getLensExceptionFromPartitionParsing(up, "garbage").getMessage(), TimePartition.getWrongUpdatePeriodMessage(up, "garbage")); getLensExceptionFromPartitionParsing(up, (Date) null); getLensExceptionFromPartitionParsing(up, (String) null); getLensExceptionFromPartitionParsing(up, ""); // parse with other update periods for (UpdatePeriod up2 : UpdatePeriod.values()) { // handles the equality case and the case where monthly-quarterly have same format strings. if (up.formatStr().equals(up2.formatStr())) { continue; } // Parsing a string representation with differnet update period should give lens exception. assertEquals(getLensExceptionFromPartitionParsing(up2, nowStr).getMessage(), TimePartition.getWrongUpdatePeriodMessage(up2, nowStr)); } }
TimePartition nowPartition = TimePartition.of(up, NOW); TimePartition tenLater = TimePartition.of(up, timeAtDiff(NOW, up, 10)); TimePartitionRange range = nowPartition.rangeUpto(tenLater); assertEquals(range, tenLater.rangeFrom(nowPartition)); assertEquals(nowPartition.singletonRange().size(), 1); String nowStr = nowPartition.getDateString(); String tenLaterStr = tenLater.getDateString(); assertEquals(TimePartitionRange.parseFrom(up, nowStr, tenLaterStr), range); assertEquals(TimePartitionRange.parseFrom(up, "[" + nowStr, tenLaterStr + ")"), range); assertEquals(TimePartitionRange.parseFrom(up, "[" + nowStr, tenLaterStr + "]"), nowPartition.rangeUpto(tenLater.next())); assertEquals(TimePartitionRange.parseFrom(up, "(" + nowStr, tenLaterStr + "]"), nowPartition.next().rangeUpto( tenLater.next())); assertEquals(TimePartitionRange.parseFrom(up, "(" + nowStr, tenLaterStr + ")"), nowPartition.next().rangeUpto(tenLater));
@Override public boolean drop(TimePartition toDrop) throws LensException { int ind = getStrictlyAfterIndex(toDrop); if (ind == 0) { return true; // nothing to do } if (ranges.get(ind - 1).getBegin().equals(toDrop)) { ranges.get(ind - 1).setBegin(toDrop.next()); } else if (ranges.get(ind - 1).getEnd().previous().equals(toDrop)) { ranges.get(ind - 1).setEnd(toDrop); } else { TimePartition end = ranges.get(ind - 1).getEnd(); ranges.get(ind - 1).setEnd(toDrop); ranges.add(ind, toDrop.next().rangeUpto(end)); } if (ranges.get(ind - 1).isEmpty()) { ranges.remove(ind - 1); } return true; }
return true; if (ranges.get(ind - 1).getEnd().equals(partition)) { added++; ranges.get(ind - 1).setEnd(partition.next()); if (partition.equals(ranges.get(ind).getBegin().previous())) { added++; ranges.get(ind).setBegin(partition); ranges.add(ind, partition.singletonRange()); break; case 2:
private LensException getLensExceptionFromPartitionParsing(UpdatePeriod up, String dateStr) { try { TimePartition.of(up, dateStr); fail("Should have thrown LensException"); } catch (LensException e) { return e; } return null; // redundant }
private TimePartition getNextPartition(TimePartition begin, TimePartition end, int increment) throws LensException { for (Date date : TimeRange.iterable(begin.partitionAtDiff(increment).getDate(), end.partitionAtDiff(increment).getDate(), begin.getUpdatePeriod(), increment)) { TimePartition value = TimePartition.of(begin.getUpdatePeriod(), date); if (!holes.contains(value)) { return value; } else { holes.remove(value); } } return null; }
public void add(String partCol, TimePartition partition) throws LensException { if (get(partCol) == null) { put(partCol, new RangesPartitionTimeline("", UpdatePeriod.values()[0], partCol)); } get(partCol).add(partition.withUpdatePeriod(UpdatePeriod.values()[0]) .rangeUpto(partition.next().withUpdatePeriod(UpdatePeriod.values()[0]))); }
@Override public boolean add(@NonNull TimePartition partition) throws LensException { if (isEmpty()) { // First partition being added first = partition; latest = partition; return true; } if (partition.before(first)) { addHolesBetween(partition, first, partition.getUpdatePeriod()); first = partition; return true; } else if (partition.after(latest)) { addHolesBetween(latest, partition, partition.getUpdatePeriod()); latest = partition; return true; } else { return holes.remove(partition); } }
@Override public boolean add(TimePartitionRange partitionRange) throws LensException { // Adding partition range to the timeline. Will have to find if any of the sub ranges // intersects. If yes, add only remaining sub ranges, else add the given range as a new sub range. int strictlyAfterIndex = getStrictlyAfterIndex(partitionRange.getBegin()); while (strictlyAfterIndex < ranges.size() && partitionRange.isValidAndNonEmpty()) { if (partitionRange.getEnd().before(ranges.get(strictlyAfterIndex).getBegin())) { // partition begin and end both are strictly before ranges[strictlyAfterIndex]. Add as new sub range. ranges.add(strictlyAfterIndex, partitionRange); partitionRange = partitionRange.getEnd().emptyRange(); break; } else { // begin is before ranges[strictlyAfterIndex], end is not. // extend ranges[strictlyAfterIndex] and add remaining range, if any. ranges.get(strictlyAfterIndex).setBegin(partitionRange.getBegin()); if (ranges.get(strictlyAfterIndex).getEnd().before(partitionRange.getEnd())) { partitionRange = ranges.get(strictlyAfterIndex).getEnd().rangeUpto(partitionRange.getEnd()); } else { // No remaining range, end was before ranges[strictlyAfterIndex].end partitionRange = ranges.get(strictlyAfterIndex).getEnd().emptyRange(); } strictlyAfterIndex++; } } if (strictlyAfterIndex == ranges.size() && partitionRange.isValidAndNonEmpty()) { ranges.add(partitionRange); } mergeRanges(); return true; }
@Override public boolean hasNext() { return current.before(end); }
public static TimePartition of(UpdatePeriod updatePeriod, String dateString) throws LensException { if (dateString == null || dateString.isEmpty()) { throw new LensException("time parition date string is null or blank"); } else { if (!updatePeriod.canParseDateString(dateString)) { throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString)); } try { return TimePartition.of(updatePeriod, updatePeriod.parse(dateString)); } catch (ParseException e) { throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString), e); } } }
/** update partition timeline cache for deletion of time partition */ public boolean updateForDeletion(String cubeTableName, String storageName, UpdatePeriod updatePeriod, Map<String, Date> timePartSpec) throws HiveException, LensException { // fail fast. All part cols mentioned in all partitions should exist. for (String partCol : timePartSpec.keySet()) { getAndFailFast(cubeTableName, storageName, updatePeriod, partCol); } boolean updated = false; for (Map.Entry<String, Date> entry : timePartSpec.entrySet()) { TimePartition part = TimePartition.of(updatePeriod, entry.getValue()); if (!partitionExistsByFilter(cubeTableName, storageName, updatePeriod, StorageConstants.getPartFilter(entry.getKey(), part.getDateString()))) { get(cubeTableName, storageName, updatePeriod, entry.getKey()).drop(part); updated = true; } } return updated; } }
public static TimePartitionRange parseFrom(UpdatePeriod updatePeriod, String from, String to) throws LensException { boolean incrementFrom = false; boolean incrementTo = false; if (from.charAt(0) == '[') { from = from.substring(1); } else if (from.charAt(0) == '(') { from = from.substring(1); incrementFrom = true; } if (to.charAt(to.length() - 1) == ']') { to = to.substring(0, to.length() - 1); incrementTo = true; } else if (to.charAt(to.length() - 1) == ')') { to = to.substring(0, to.length() - 1); } TimePartition fromPartition = TimePartition.of(updatePeriod, from); TimePartition toPartition = TimePartition.of(updatePeriod, to); if (incrementFrom) { fromPartition = fromPartition.next(); } if (incrementTo) { toPartition = toPartition.next(); } return new TimePartitionRange(fromPartition, toPartition); }