public double getAllTableWeights(ImmutableMap<String, Double> tableWeights) { double weight = 0; Map<String, Double> tblWithoutDBWeghts = new HashMap<>(); for (Map.Entry<String, Double> entry : tableWeights.entrySet()) { tblWithoutDBWeghts.put(entry.getKey().substring(entry.getKey().indexOf('.') + 1), entry.getValue()); } for (String tblName : getStorageTables()) { Double tblWeight = tblWithoutDBWeghts.get(tblName); if (tblWeight != null) { weight += tblWeight; } } return weight; } }
public double getAllTableWeights(ImmutableMap<String, Double> tableWeights) { double weight = 0; Map<String, Double> tblWithoutDBWeghts = new HashMap<>(); for (Map.Entry<String, Double> entry : tableWeights.entrySet()) { tblWithoutDBWeghts.put(entry.getKey().substring(entry.getKey().indexOf('.') + 1), entry.getValue()); } for (String tblName : getStorageTables()) { Double tblWeight = tblWithoutDBWeghts.get(tblName); if (tblWeight != null) { weight += tblWeight; } } return weight; } }
/** * Truncates partitions in {@link #rangeToPartitions} such that only partitions belonging to * the passed undatePeriod are retained. * @param updatePeriod */ private void truncatePartitions(UpdatePeriod updatePeriod) { Iterator<Map.Entry<TimeRange, Set<FactPartition>>> rangeItr = rangeToPartitions.entrySet().iterator(); while (rangeItr.hasNext()) { Map.Entry<TimeRange, Set<FactPartition>> rangeEntry = rangeItr.next(); rangeEntry.getValue().removeIf(factPartition -> !factPartition.getPeriod().equals(updatePeriod)); rangeEntry.getValue().forEach(factPartition -> { factPartition.getStorageTables().remove(storageTable); factPartition.getStorageTables().add(resolvedName); }); if (rangeEntry.getValue().isEmpty()) { rangeItr.remove(); } } }
/** * Truncates partitions in {@link #rangeToPartitions} such that only partitions belonging to * the passed undatePeriod are retained. * @param updatePeriod */ private void truncatePartitions(UpdatePeriod updatePeriod) { Iterator<Map.Entry<TimeRange, Set<FactPartition>>> rangeItr = rangeToPartitions.entrySet().iterator(); while (rangeItr.hasNext()) { Map.Entry<TimeRange, Set<FactPartition>> rangeEntry = rangeItr.next(); rangeEntry.getValue().removeIf(factPartition -> !factPartition.getPeriod().equals(updatePeriod)); rangeEntry.getValue().forEach(factPartition -> { factPartition.getStorageTables().remove(storageTable); factPartition.getStorageTables().add(resolvedName); }); if (rangeEntry.getValue().isEmpty()) { rangeItr.remove(); } } }
for (String table : part.getStorageTables()) { invertedMap.computeIfAbsent(table, k -> new TreeSet<>()).add(part);
private void updatePartitionStorage(FactPartition part) throws LensException { try { if (getCubeMetastoreClient().factPartitionExists(fact, part, storageTable)) { part.getStorageTables().add(storageTable); part.setFound(true); } } catch (HiveException e) { log.warn("Hive exception while getting storage table partition", e); } }
for (String table : part.getStorageTables()) { invertedMap.computeIfAbsent(table, k -> new TreeSet<>()).add(part);
private void updatePartitionStorage(FactPartition part) throws LensException { try { if (getCubeMetastoreClient().factPartitionExists(fact, part, storageTable)) { part.getStorageTables().add(storageTable); part.setFound(true); } } catch (HiveException e) { log.warn("Hive exception while getting storage table partition", e); } }
/** * Partition should not be used a indicative of the class itself. * New Fact partition created includes more final partitions with that creation. * @return */ public FactPartition withoutContaining() { return new FactPartition(this.getPartCol(), this.getPartSpec(), this.getPeriod(), null, this .getPartFormat(), this.getStorageTables()); } public FactPartition(String partCol, TimePartition timePartition) {
/** * Partition should not be used a indicative of the class itself. * New Fact partition created includes more final partitions with that creation. * @return */ public FactPartition withoutContaining() { return new FactPartition(this.getPartCol(), this.getPartSpec(), this.getPeriod(), null, this .getPartFormat(), this.getStorageTables()); } public FactPartition(String partCol, TimePartition timePartition) {
public FactPartition next() throws LensException { return new FactPartition(getPartCol(), getTimePartition().next(), getContainingPart(), getStorageTables()); }
public FactPartition previous() throws LensException { return new FactPartition(getPartCol(), getTimePartition().previous(), getContainingPart(), getStorageTables()); }
public FactPartition next() throws LensException { return new FactPartition(getPartCol(), getTimePartition().next(), getContainingPart(), getStorageTables()); }
public FactPartition previous() throws LensException { return new FactPartition(getPartCol(), getTimePartition().previous(), getContainingPart(), getStorageTables()); }
FactPartition part = new FactPartition(partCol, fromDate, maxInterval, null, partWhereClauseFormat); partitions.add(part); part.getStorageTables().add(storageTable); part = new FactPartition(partCol, toDate, maxInterval, null, partWhereClauseFormat); partitions.add(part); part.getStorageTables().add(storageTable); this.participatingUpdatePeriods.add(maxInterval); log.info("Added continuous fact partition for storage table {}", storageName); FactPartition part = new FactPartition(partCol, dt, maxInterval, null, partWhereClauseFormat); updatePartitionStorage(part); log.debug("Storage tables containing Partition {} are: {}", part, part.getStorageTables()); if (part.isFound()) { log.debug("Adding existing partition {}", part); this.participatingUpdatePeriods.add(maxInterval); partitions.add(part); part.getStorageTables().add(storageTable);
FactPartition part = new FactPartition(partCol, fromDate, maxInterval, null, partWhereClauseFormat); partitions.add(part); part.getStorageTables().add(storageTable); part = new FactPartition(partCol, toDate, maxInterval, null, partWhereClauseFormat); partitions.add(part); part.getStorageTables().add(storageTable); this.participatingUpdatePeriods.add(maxInterval); log.info("Added continuous fact partition for storage table {}", storageName); FactPartition part = new FactPartition(partCol, dt, maxInterval, null, partWhereClauseFormat); updatePartitionStorage(part); log.debug("Storage tables containing Partition {} are: {}", part, part.getStorageTables()); if (part.isFound()) { log.debug("Adding existing partition {}", part); this.participatingUpdatePeriods.add(maxInterval); partitions.add(part); part.getStorageTables().add(storageTable);