public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod, Map<String, Date> partitionTimestamp, Map<String, String> partSpec) throws HiveException, LensException { String storageTableName = getStorageTableName(factName, storageName, updatePeriod); return partitionExists(storageTableName, updatePeriod, partitionTimestamp, partSpec); }
public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod, Map<String, Date> partitionTimestamp, Map<String, String> partSpec) throws HiveException, LensException { String storageTableName = getStorageTableName(factName, storageName, updatePeriod); return partitionExists(storageTableName, updatePeriod, partitionTimestamp, partSpec); }
private void dropHiveTablesForStorage(String factName, String storage) throws LensException { CubeFactTable cft = getCubeFactTable(factName); Set<String> droppedTables = new HashSet<>(); for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) { UpdatePeriod updatePeriod = (UpdatePeriod) updatePeriodEntry.getKey(); String storageTableName = getStorageTableName(factName, storage, updatePeriod); if (!droppedTables.contains(storageTableName)) { dropHiveTable(storageTableName); } droppedTables.add(storageTableName); } }
private void dropHiveTablesForStorage(String factName, String storage) throws LensException{ CubeFactTable cft = getCubeFactTable(factName); Set<String> droppedTables = new HashSet<>(); for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) { UpdatePeriod updatePeriod = (UpdatePeriod) updatePeriodEntry.getKey(); String storageTableName = getStorageTableName(factName, storage, updatePeriod); if (!droppedTables.contains(storageTableName)) { dropHiveTable(storageTableName); } droppedTables.add(storageTableName); } } // updateFact will be false when fact is fully dropped
public void updatePartitions(String factOrDimtableName, String storageName, Map<UpdatePeriod, List<Partition>> partitions) throws HiveException, InvalidOperationException, LensException { for (Map.Entry entry : partitions.entrySet()) { List<Partition> partitionsToAlter = Lists.newArrayList(); partitionsToAlter.addAll((List<Partition>) entry.getValue()); String storageTableName = getStorageTableName(factOrDimtableName, storageName, (UpdatePeriod) entry.getKey()); partitionsToAlter.addAll( getAllLatestPartsEquivalentTo(factOrDimtableName, storageTableName, (List<Partition>) entry.getValue())); getStorage(storageName).updatePartitions(storageTableName, getClient(), factOrDimtableName, partitionsToAlter); } }
public void updatePartitions(String factOrDimtableName, String storageName, Map<UpdatePeriod, List<Partition>> partitions) throws HiveException, InvalidOperationException, LensException { for (Map.Entry entry : partitions.entrySet()) { List<Partition> partitionsToAlter = Lists.newArrayList(); partitionsToAlter.addAll((List<Partition>) entry.getValue()); String storageTableName = getStorageTableName(factOrDimtableName, storageName, (UpdatePeriod) entry.getKey()); partitionsToAlter.addAll( getAllLatestPartsEquivalentTo(factOrDimtableName, storageTableName, (List<Partition>) entry.getValue())); getStorage(storageName).updatePartitions(storageTableName, getClient(), factOrDimtableName, partitionsToAlter); } }
/** * Returns storage table names for a storage. * Note: If each update period in the storage has a different storage table, this method will return N Storage Tables * where N is the number of update periods in the storage (LENS-1386) * * @param factTable * @param storage * @return * @throws LensException */ public Set<String> getStorageTables(FactTable factTable, String storage) throws LensException { Set<String> uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) { String factName = factTable.getSourceFactName(); uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod)); } return uniqueStorageTables; }
/** * Returns storage table names for a storage. * Note: If each update period in the storage has a different storage table, this method will return N Storage Tables * where N is the number of update periods in the storage (LENS-1386) * * @param factTable * @param storage * @return * @throws LensException */ public Set<String> getStorageTables(FactTable factTable, String storage) throws LensException { Set<String> uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) { String factName = factTable.getSourceFactName(); uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod)); } return uniqueStorageTables; }
private Set<String> getValidStorageTableNames() throws LensException { if (!validUpdatePeriods.isEmpty()) { // In this case skip invalid update periods and get storage tables only for valid ones. Set<String> uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : validUpdatePeriods) { uniqueStorageTables.add( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, updatePeriod) ); } return uniqueStorageTables; } else { //Get all storage tables. return getCubeMetastoreClient().getStorageTables(fact, storageName); } }
private Set<String> getValidStorageTableNames() throws LensException { if (!validUpdatePeriods.isEmpty()) { // In this case skip invalid update periods and get storage tables only for valid ones. Set<String> uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : validUpdatePeriods) { uniqueStorageTables.add( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, updatePeriod) ); } return uniqueStorageTables; } else { //Get all storage tables. return getCubeMetastoreClient().getStorageTables(fact, storageName); } }
private Collection<StorageCandidate> getPeriodSpecificStorageCandidates() throws LensException { if (periodSpecificStorageCandidates == null) { List<StorageCandidate> periodSpecificScList = new ArrayList<>(participatingUpdatePeriods.size()); StorageCandidate updatePeriodSpecificSc; for (UpdatePeriod period : participatingUpdatePeriods) { updatePeriodSpecificSc = copy(); updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, period)); updatePeriodSpecificSc.truncatePartitions(period); periodSpecificScList.add(updatePeriodSpecificSc); } periodSpecificStorageCandidates = periodSpecificScList; } return periodSpecificStorageCandidates; }
private Collection<StorageCandidate> getPeriodSpecificStorageCandidates() throws LensException { if (periodSpecificStorageCandidates == null) { List<StorageCandidate> periodSpecificScList = new ArrayList<>(participatingUpdatePeriods.size()); StorageCandidate updatePeriodSpecificSc; for (UpdatePeriod period : participatingUpdatePeriods) { updatePeriodSpecificSc = copy(); updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, period)); updatePeriodSpecificSc.isStorageTblsAtUpdatePeriodLevel = false; updatePeriodSpecificSc.truncatePartitions(period); periodSpecificScList.add(updatePeriodSpecificSc); } periodSpecificStorageCandidates = periodSpecificScList; } return periodSpecificStorageCandidates; }
Map<UpdatePeriod, String> updatePeriodTableName = new HashMap<>(); for (UpdatePeriod updatePeriod : getFactTable(fact).getUpdatePeriods().get(storage)) { String storageTableName = getStorageTableName(fact, storage, updatePeriod); updatePeriodTableName.put(updatePeriod, storageTableName); Table storageTable = getTable(storageTableName);
private Date getStorageTableEndDate(UpdatePeriod interval) throws LensException { if (!isStorageTblsAtUpdatePeriodLevel) { //In this case the start time and end time is at Storage Level and will be same for all update periods. return this.endTime; } return getCubeMetastoreClient().getStorageTableEndDate( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact); }
private Date getStorageTableEndDate(UpdatePeriod interval) throws LensException { if (!isStorageTblsAtUpdatePeriodLevel) { //In this case the start time and end time is at Storage Level and will be same for all update periods. return this.endTime; } return getCubeMetastoreClient().getStorageTableEndDate( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact); }
private Date getStorageTableStartDate(UpdatePeriod interval) throws LensException { if (!isStorageTblsAtUpdatePeriodLevel) { //In this case the start time and end time is at Storage Level and will be same for all update periods. return this.startTime; } return getCubeMetastoreClient().getStorageTableStartDate( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact); }
private Date getStorageTableStartDate(UpdatePeriod interval) throws LensException { if (!isStorageTblsAtUpdatePeriodLevel) { //In this case the start time and end time is at Storage Level and will be same for all update periods. return this.startTime; } return getCubeMetastoreClient().getStorageTableStartDate( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact); }
public boolean partitionExistsByFilter(String storageTableName, String filter) throws LensException { int parts; Table tbl; try { tbl = getTable(storageTableName); } catch (Exception e) { return false; } try { parts = getClient().getNumPartitionsByFilter(tbl, filter); } catch (Exception e) { throw new LensException("Could not find partitions for given filter", e); } return parts > 0; }
public boolean partitionExistsByFilter(String storageTableName, String filter) throws LensException { int parts; Table tbl; try { tbl = getTable(storageTableName); } catch (Exception e) { return false; } try { parts = getClient().getNumPartitionsByFilter(tbl, filter); } catch (Exception e) { throw new LensException("Could not find partitions for given filter", e); } return parts > 0; }
@Override public void updatePartition(LensSessionHandle sessionid, String tblName, String storageName, XPartition xPartition) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient client = getClient(sessionid); String storageTableName = client .getStorageTableName(tblName, storageName, UpdatePeriod.valueOf(xPartition.getUpdatePeriod().name())); Partition existingPartition = client.getPartitionByFilter(storageTableName, StorageConstants.getPartFilter(JAXBUtils.getFullPartSpecAsMap(xPartition))); JAXBUtils.updatePartitionFromXPartition(existingPartition, xPartition); client.updatePartition(tblName, storageName, existingPartition, UpdatePeriod.valueOf(xPartition.getUpdatePeriod().value())); } catch (HiveException | ClassNotFoundException | InvalidOperationException | UnsupportedOperationException exc) { throw new LensException(exc); } }