public FactPartition(String partCol, TimePartition timePartition, FactPartition containingPart, Set<String> storageTables) { this(partCol, timePartition.getDate(), timePartition.getUpdatePeriod(), containingPart, null, storageTables); }
public FactPartition(String partCol, TimePartition timePartition, FactPartition containingPart, Set<String> storageTables) { this(partCol, timePartition.getDate(), timePartition.getUpdatePeriod(), containingPart, null, storageTables); }
public TimePartition withUpdatePeriod(UpdatePeriod period) throws LensException { return TimePartition.of(period, getDate()); }
public TimePartition withUpdatePeriod(UpdatePeriod period) throws LensException { return TimePartition.of(period, getDate()); }
/** wrapper on latest data */ public Date getLatestDate() { return latest() == null ? null : latest().getDate(); }
/** wrapper on latest data */ public Date getLatestDate() { return latest() == null ? null : latest().getDate(); }
private TimePartition getNextPartition(TimePartition begin, TimePartition end, int increment) throws LensException { for (Date date : TimeRange.iterable(begin.partitionAtDiff(increment).getDate(), end.partitionAtDiff(increment).getDate(), begin.getUpdatePeriod(), increment)) { TimePartition value = TimePartition.of(begin.getUpdatePeriod(), date); if (!holes.contains(value)) { return value; } else { holes.remove(value); } } return null; }
private TimePartition getNextPartition(TimePartition begin, TimePartition end, int increment) throws LensException { for (Date date : TimeRange.iterable(begin.partitionAtDiff(increment).getDate(), end.partitionAtDiff(increment).getDate(), begin.getUpdatePeriod(), increment)) { TimePartition value = TimePartition.of(begin.getUpdatePeriod(), date); if (!holes.contains(value)) { return value; } else { holes.remove(value); } } return null; }
private void assertLatestForRegions(String storageTableName, Map<String, Date> expectedLatestValues) throws HiveException, LensException { List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), expectedLatestValues.size()); for (Partition part : parts) { assertEquals(MetastoreUtil.getLatestTimeStampFromPartition(part, getDatePartitionKey()), TimePartition.of(HOURLY, expectedLatestValues.get(part.getSpec().get("region"))).getDate()); } }
private void assertTimeline(EndsAndHolesPartitionTimeline endsAndHolesPartitionTimeline, StoreAllPartitionTimeline storeAllPartitionTimeline, UpdatePeriod updatePeriod, Date first, Date latest, Date... holes) throws LensException { TimePartition firstPart = TimePartition.of(updatePeriod, first); TimePartition latestPart = TimePartition.of(updatePeriod, latest); assertEquals(endsAndHolesPartitionTimeline.getFirst(), firstPart); assertEquals(endsAndHolesPartitionTimeline.getLatest(), TimePartition.of(updatePeriod, latest)); assertEquals(endsAndHolesPartitionTimeline.getHoles().size(), holes.length); for (Date date : holes) { assertTrue(endsAndHolesPartitionTimeline.getHoles().contains(TimePartition.of(updatePeriod, date))); } TreeSet<TimePartition> partitions = new TreeSet<>(); for (Date dt : TimeRange.iterable(firstPart.getDate(), latestPart.next().getDate(), updatePeriod, 1)) { partitions.add(TimePartition.of(updatePeriod, dt)); } for (Date holeDate : holes) { partitions.remove(TimePartition.of(updatePeriod, holeDate)); } assertEquals(storeAllPartitionTimeline.getAllPartitions(), partitions); }