/** * Returns storage table names for a storage. * Note: If each update period in the storage has a different storage table, this method will return N Storage Tables * where N is the number of update periods in the storage (LENS-1386) * * @param factTable * @param storage * @return * @throws LensException */ public Set<String> getStorageTables(FactTable factTable, String storage) throws LensException { Set<String> uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) { String factName = factTable.getSourceFactName(); uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod)); } return uniqueStorageTables; }
/** * Returns storage table names for a storage. * Note: If each update period in the storage has a different storage table, this method will return N Storage Tables * where N is the number of update periods in the storage (LENS-1386) * * @param factTable * @param storage * @return * @throws LensException */ public Set<String> getStorageTables(FactTable factTable, String storage) throws LensException { Set<String> uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) { String factName = factTable.getSourceFactName(); uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod)); } return uniqueStorageTables; }
private Set<String> getValidStorageTableNames() throws LensException { if (!validUpdatePeriods.isEmpty()) { // In this case skip invalid update periods and get storage tables only for valid ones. Set<String> uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : validUpdatePeriods) { uniqueStorageTables.add( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, updatePeriod) ); } return uniqueStorageTables; } else { //Get all storage tables. return getCubeMetastoreClient().getStorageTables(fact, storageName); } }
/** extract storage name from fact and storage table name. String operation */ private String extractStorageName(FactTable fact, String storageTableName) throws LensException { int ind = storageTableName.lastIndexOf(fact.getSourceFactName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); } String name = storageTableName.substring(0, ind - StorageConstants.STORGAE_SEPARATOR.length()); for (String storageName : fact.getStorages()) { if (name.equalsIgnoreCase(storageName)) { return storageName; } } throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); }
/** extract storage name from fact and storage table name. String operation */ private String extractStorageName(FactTable fact, String storageTableName) throws LensException { int ind = storageTableName.lastIndexOf(fact.getSourceFactName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); } String name = storageTableName.substring(0, ind - StorageConstants.STORGAE_SEPARATOR.length()); for (String storageName : fact.getStorages()) { if (name.equalsIgnoreCase(storageName)) { return storageName; } } throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); }
private Set<String> getValidStorageTableNames() throws LensException { if (!validUpdatePeriods.isEmpty()) { // In this case skip invalid update periods and get storage tables only for valid ones. Set<String> uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : validUpdatePeriods) { uniqueStorageTables.add( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, updatePeriod) ); } return uniqueStorageTables; } else { //Get all storage tables. return getCubeMetastoreClient().getStorageTables(fact, storageName); } }
private Collection<StorageCandidate> getPeriodSpecificStorageCandidates() throws LensException { if (periodSpecificStorageCandidates == null) { List<StorageCandidate> periodSpecificScList = new ArrayList<>(participatingUpdatePeriods.size()); StorageCandidate updatePeriodSpecificSc; for (UpdatePeriod period : participatingUpdatePeriods) { updatePeriodSpecificSc = copy(); updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, period)); updatePeriodSpecificSc.truncatePartitions(period); periodSpecificScList.add(updatePeriodSpecificSc); } periodSpecificStorageCandidates = periodSpecificScList; } return periodSpecificStorageCandidates; }
private Collection<StorageCandidate> getPeriodSpecificStorageCandidates() throws LensException { if (periodSpecificStorageCandidates == null) { List<StorageCandidate> periodSpecificScList = new ArrayList<>(participatingUpdatePeriods.size()); StorageCandidate updatePeriodSpecificSc; for (UpdatePeriod period : participatingUpdatePeriods) { updatePeriodSpecificSc = copy(); updatePeriodSpecificSc.setResolvedName(getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, period)); updatePeriodSpecificSc.isStorageTblsAtUpdatePeriodLevel = false; updatePeriodSpecificSc.truncatePartitions(period); periodSpecificScList.add(updatePeriodSpecificSc); } periodSpecificStorageCandidates = periodSpecificScList; } return periodSpecificStorageCandidates; }
private Date getStorageTableEndDate(UpdatePeriod interval) throws LensException { if (!isStorageTblsAtUpdatePeriodLevel) { //In this case the start time and end time is at Storage Level and will be same for all update periods. return this.endTime; } return getCubeMetastoreClient().getStorageTableEndDate( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact); }
private Date getStorageTableStartDate(UpdatePeriod interval) throws LensException { if (!isStorageTblsAtUpdatePeriodLevel) { //In this case the start time and end time is at Storage Level and will be same for all update periods. return this.startTime; } return getCubeMetastoreClient().getStorageTableStartDate( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact); }
private Date getStorageTableStartDate(UpdatePeriod interval) throws LensException { if (!isStorageTblsAtUpdatePeriodLevel) { //In this case the start time and end time is at Storage Level and will be same for all update periods. return this.startTime; } return getCubeMetastoreClient().getStorageTableStartDate( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact); }
private Date getStorageTableEndDate(UpdatePeriod interval) throws LensException { if (!isStorageTblsAtUpdatePeriodLevel) { //In this case the start time and end time is at Storage Level and will be same for all update periods. return this.endTime; } return getCubeMetastoreClient().getStorageTableEndDate( getCubeMetastoreClient().getStorageTableName(fact.getSourceFactName(), storageName, interval), fact); }
public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeQueryContext) throws LensException { this.cube = cube; this.fact = fact; this.cubeQueryContext = cubeQueryContext; if ((getCube() == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.storageName = storageName; this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getSourceFactName(), storageName); this.name = getFact().getName(); this.processTimePartCol = getConf().get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String formatStr = getConf().get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); if (formatStr != null) { this.partWhereClauseFormat = new SimpleDateFormat(formatStr); } completenessPartCol = getConf().get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); completenessThreshold = getConf() .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); Set<String> storageTblNames = getCubeMetastoreClient().getStorageTables(fact, storageName); isStorageTblsAtUpdatePeriodLevel = storageTblNames.size() > 1 || !storageTblNames.iterator().next().equalsIgnoreCase(storageTable); setStorageStartAndEndDate(); }
public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeQueryContext) throws LensException { this.cube = cube; this.fact = fact; this.cubeQueryContext = cubeQueryContext; if ((getCube() == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.storageName = storageName; this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getSourceFactName(), storageName); this.name = getFact().getName(); this.processTimePartCol = getConf().get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String formatStr = getConf().get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); if (formatStr != null) { this.partWhereClauseFormat = new SimpleDateFormat(formatStr); } completenessPartCol = getConf().get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); completenessThreshold = getConf() .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); Set<String> storageTblNames = getCubeMetastoreClient().getStorageTables(fact, storageName); isStorageTblsAtUpdatePeriodLevel = storageTblNames.size() > 1 || !storageTblNames.iterator().next().equalsIgnoreCase(storageTable); setStorageStartAndEndDate(); }
/** extract storage name and check in timeline cache for existance */ public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); return partitionTimelineCache.partitionTimeExists(fact.getSourceFactName(), storage, part.getPeriod(), part.getPartCol(), part.getPartSpec()); }
for (String storage : fact.getStorages()) { for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) { PartitionTimeline timeline = partitionTimelineCache.get(fact.getSourceFactName(), storage, updatePeriod, partCol); if (timeline != null) {// this storage table is partitioned by partCol or not.
for (String storage : fact.getStorages()) { for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) { PartitionTimeline timeline = partitionTimelineCache.get(fact.getSourceFactName(), storage, updatePeriod, partCol); if (timeline != null) {// this storage table is partitioned by partCol or not.
/** extract storage name and check in timeline cache for existance */ public boolean factPartitionExists(FactTable fact, FactPartition part, String storageTableName) throws HiveException, LensException { String storage = extractStorageName(fact, storageTableName); return partitionTimelineCache.partitionTimeExists(fact.getSourceFactName(), storage, part.getPeriod(), part.getPartCol(), part.getPartSpec()); }
TimeRange fallBackRange = getFallbackRange(prevRange, this.getFact().getSourceFactName(), cubeQueryContext); log.info("No partitions for range:{}. fallback range: {}", timeRange, fallBackRange); if (fallBackRange == null) {
if (!client.partColExists(sc.getFact(), sc.getStorageName(), range.getPartitionColumn())) { pruningCauseForThisTimeRange = partitionColumnsMissing(range.getPartitionColumn()); TimeRange fallBackRange = getFallbackRange(range, sc.getFact().getSourceFactName(), cubeql); while (fallBackRange != null) { pruningCauseForThisTimeRange = null; fallBackRange.getPartitionColumn())) { pruningCauseForThisTimeRange = partitionColumnsMissing(fallBackRange.getPartitionColumn()); fallBackRange = getFallbackRange(fallBackRange, sc.getFact().getSourceFactName(), cubeql); } else { if (!sc.isPartiallyValidForTimeRange(fallBackRange)) {