/** * Truncates partitions in {@link #rangeToPartitions} such that only partitions belonging to * the passed undatePeriod are retained. * @param updatePeriod */ private void truncatePartitions(UpdatePeriod updatePeriod) { Iterator<Map.Entry<TimeRange, Set<FactPartition>>> rangeItr = rangeToPartitions.entrySet().iterator(); while (rangeItr.hasNext()) { Map.Entry<TimeRange, Set<FactPartition>> rangeEntry = rangeItr.next(); rangeEntry.getValue().removeIf(factPartition -> !factPartition.getPeriod().equals(updatePeriod)); rangeEntry.getValue().forEach(factPartition -> { factPartition.getStorageTables().remove(storageTable); factPartition.getStorageTables().add(resolvedName); }); if (rangeEntry.getValue().isEmpty()) { rangeItr.remove(); } } }
/** * Truncates partitions in {@link #rangeToPartitions} such that only partitions belonging to * the passed undatePeriod are retained. * @param updatePeriod */ private void truncatePartitions(UpdatePeriod updatePeriod) { Iterator<Map.Entry<TimeRange, Set<FactPartition>>> rangeItr = rangeToPartitions.entrySet().iterator(); while (rangeItr.hasNext()) { Map.Entry<TimeRange, Set<FactPartition>> rangeEntry = rangeItr.next(); rangeEntry.getValue().removeIf(factPartition -> !factPartition.getPeriod().equals(updatePeriod)); rangeEntry.getValue().forEach(factPartition -> { factPartition.getStorageTables().remove(storageTable); factPartition.getStorageTables().add(resolvedName); }); if (rangeEntry.getValue().isEmpty()) { rangeItr.remove(); } } }
public TimePartition getTimePartition() throws LensException { return TimePartition.of(getPeriod(), getPartSpec()); }
public TimePartition getTimePartition() throws LensException { return TimePartition.of(getPeriod(), getPartSpec()); }
private FactPartition mockFactPartition(UpdatePeriod mockPeriod, ImmutableMap<String, Double> tableWeights, double mockAllTableWeight) { FactPartition fp = mock(FactPartition.class); when(fp.getPeriod()).thenReturn(mockPeriod); when(fp.getAllTableWeights(tableWeights)).thenReturn(mockAllTableWeight); return fp; }
allTableWeights = 1; cost += allTableWeights * getNormalizedUpdatePeriodCost(partition.getPeriod(), driver);
allTableWeights = 1; cost += allTableWeights * getNormalizedUpdatePeriodCost(partition.getPeriod(), driver);
/** extract storage name and check in timeline cache for existance */ public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); return partitionTimelineCache.partitionTimeExists(fact.getSourceFactName(), storage, part.getPeriod(), part.getPartCol(), part.getPartSpec()); }
/** * Partition should not be used a indicative of the class itself. * New Fact partition created includes more final partitions with that creation. * @return */ public FactPartition withoutContaining() { return new FactPartition(this.getPartCol(), this.getPartSpec(), this.getPeriod(), null, this .getPartFormat(), this.getStorageTables()); } public FactPartition(String partCol, TimePartition timePartition) {
/** extract storage name and check in timeline cache for existance */ public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); return partitionTimelineCache.partitionTimeExists(fact.getSourceFactName(), storage, part.getPeriod(), part.getPartCol(), part.getPartSpec()); }
/** * Partition should not be used a indicative of the class itself. * New Fact partition created includes more final partitions with that creation. * @return */ public FactPartition withoutContaining() { return new FactPartition(this.getPartCol(), this.getPartSpec(), this.getPeriod(), null, this .getPartFormat(), this.getStorageTables()); } public FactPartition(String partCol, TimePartition timePartition) {
for (FactPartition pPart : processTimeParts) { log.debug("Looking for finer partitions in pPart: {}", pPart); for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) { FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart, partWhereClauseFormat); updatePartitionStorage(innerPart);
for (FactPartition pPart : processTimeParts) { log.debug("Looking for finer partitions in pPart: {}", pPart); for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) { FactPartition innerPart = new FactPartition(partCol, date, pPart.getPeriod(), pPart, partWhereClauseFormat); updatePartitionStorage(innerPart);