public TimePartition withUpdatePeriod(UpdatePeriod period) throws LensException { return TimePartition.of(period, getDate()); }
private LensException getLensExceptionFromPartitionParsing(UpdatePeriod up, String dateStr) { try { TimePartition.of(up, dateStr); fail("Should have thrown LensException"); } catch (LensException e) { return e; } return null; // redundant }
private LensException getLensExceptionFromPartitionParsing(UpdatePeriod up, Date date) { try { TimePartition.of(up, date); fail("Should have thrown LensException"); } catch (LensException e) { return e; } return null; // redundant }
/** update partition timeline cache for addition of time partition */ public void updateForAddition(String cubeTableName, String storageName, UpdatePeriod updatePeriod, Map<String, TreeSet<Date>> timePartSpec) throws HiveException, LensException { // fail fast. All part cols mentioned in all partitions should exist. for (String partCol : timePartSpec.keySet()) { getAndFailFast(cubeTableName, storageName, updatePeriod, partCol); } for (Map.Entry<String, TreeSet<Date>> entry : timePartSpec.entrySet()) { for (Date dt : entry.getValue()) { get(cubeTableName, storageName, updatePeriod, entry.getKey()).add(TimePartition.of(updatePeriod, dt)); } } }
@Override public boolean initFromProperties(Map<String, String> properties) throws LensException { allPartitions.clear(); String partitionsStr = MetastoreUtil.getNamedStringValue(properties, "partitions"); if (partitionsStr == null) { return true; } for (String s : StringUtils.split(partitionsStr, ",")) { add(TimePartition.of(getUpdatePeriod(), s)); } return true; }
@Override public boolean initFromProperties(Map<String, String> properties) throws LensException { allPartitions.clear(); String partitionsStr = MetastoreUtil.getNamedStringValue(properties, "partitions"); if (partitionsStr == null) { return true; } for (String s : StringUtils.split(partitionsStr, ",")) { add(TimePartition.of(getUpdatePeriod(), s)); } return true; }
public TimePartition getTimePartition() throws LensException { return TimePartition.of(getPeriod(), getPartSpec()); }
@Test(expectedExceptions = LensException.class) public void testTimeRangeCreationWithDifferentUpdatePeriod() throws LensException { // begin and end partitions should have same update period for range creation to succeed. TimePartition.of(UpdatePeriod.HOURLY, NOW).rangeUpto(TimePartition.of(UpdatePeriod.DAILY, NOW)); }
/** check partition existence in the appropriate timeline if it exists */ public boolean partitionTimeExists(String name, String storage, UpdatePeriod period, String partCol, Date partSpec) throws HiveException, LensException { return get(name, storage, period, partCol) != null && get(name, storage, period, partCol).exists(TimePartition.of(period, partSpec)); }
/** check partition existence in the appropriate timeline if it exists */ public boolean partitionTimeExists(String name, String storage, UpdatePeriod period, String partCol, Date partSpec) throws HiveException, LensException { return get(name, storage, period, partCol) != null && get(name, storage, period, partCol).exists(TimePartition.of(period, partSpec)); }
private TimePartition[] toPartitionArray(UpdatePeriod updatePeriod, int... offsets) throws LensException { TimePartition[] values = new TimePartition[offsets.length]; for (int i = 0; i < offsets.length; i++) { values[i] = TimePartition.of(updatePeriod, getDateWithOffset(updatePeriod, offsets[i])); } return values; }
@Test(expectedExceptions = LensException.class) public void testPartitionRangeValidity() throws LensException { // begin and end partitions should follow begin <= end TimePartition.of(UpdatePeriod.HOURLY, NOW) .rangeFrom(TimePartition.of(UpdatePeriod.HOURLY, timeAtDiff(NOW, UpdatePeriod.HOURLY, 10))); }
private TimePartition getNextPartition(TimePartition begin, TimePartition end, int increment) throws LensException { for (Date date : TimeRange.iterable(begin.partitionAtDiff(increment).getDate(), end.partitionAtDiff(increment).getDate(), begin.getUpdatePeriod(), increment)) { TimePartition value = TimePartition.of(begin.getUpdatePeriod(), date); if (!holes.contains(value)) { return value; } else { holes.remove(value); } } return null; }
private TimePartition getNextPartition(TimePartition begin, TimePartition end, int increment) throws LensException { for (Date date : TimeRange.iterable(begin.partitionAtDiff(increment).getDate(), end.partitionAtDiff(increment).getDate(), begin.getUpdatePeriod(), increment)) { TimePartition value = TimePartition.of(begin.getUpdatePeriod(), date); if (!holes.contains(value)) { return value; } else { holes.remove(value); } } return null; }
private void addHolesBetween(TimePartition begin, TimePartition end, UpdatePeriod updatePeriod) throws LensException { for (Date date : TimeRange.iterable(begin.next().getDate(), end.getDate(), updatePeriod, 1)) { addHole(TimePartition.of(updatePeriod, date)); } }
private void addHolesBetween(TimePartition begin, TimePartition end, UpdatePeriod updatePeriod) throws LensException { for (Date date : TimeRange.iterable(begin.next().getDate(), end.getDate(), updatePeriod, 1)) { addHole(TimePartition.of(updatePeriod, date)); } }
private void assertLatestForRegions(String storageTableName, Map<String, Date> expectedLatestValues) throws HiveException, LensException { List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), expectedLatestValues.size()); for (Partition part : parts) { assertEquals(MetastoreUtil.getLatestTimeStampFromPartition(part, getDatePartitionKey()), TimePartition.of(HOURLY, expectedLatestValues.get(part.getSpec().get("region"))).getDate()); } }
public static TimePartition of(UpdatePeriod updatePeriod, String dateString) throws LensException { if (dateString == null || dateString.isEmpty()) { throw new LensException("time parition date string is null or blank"); } else { if (!updatePeriod.canParseDateString(dateString)) { throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString)); } try { return TimePartition.of(updatePeriod, updatePeriod.parse(dateString)); } catch (ParseException e) { throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString), e); } } }
public static TimePartition of(UpdatePeriod updatePeriod, String dateString) throws LensException { if (dateString == null || dateString.isEmpty()) { throw new LensException("time parition date string is null or blank"); } else { if (!updatePeriod.canParseDateString(dateString)) { throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString)); } try { return TimePartition.of(updatePeriod, updatePeriod.parse(dateString)); } catch (ParseException e) { throw new LensException(getWrongUpdatePeriodMessage(updatePeriod, dateString), e); } } }