public String getStorageTableName(String factOrDimTableName, String storage, UpdatePeriod updatePeriod) throws LensException { return MetastoreUtil.getFactOrDimtableStorageTableName(factOrDimTableName, getStorageTablePrefixFromStorage(factOrDimTableName, storage, updatePeriod)); } }
/** * @return the timedDimension */ public String getTimedDimension() { return getProperties().get(MetastoreUtil.getDimTimedDimensionKey(getName())); }
private Set<String> getAllTablesForStorage(LensSessionHandle sessionHandle, String fact, String storageName) throws LensException { Set<String> storageTableNames = new HashSet<>(); if (getClient(sessionHandle).isFactTable(fact)) { CubeFactTable cft = getClient(sessionHandle).getCubeFactTable(fact); Map<UpdatePeriod, String> storageMap = cft.getStoragePrefixUpdatePeriodMap().get(storageName); for (Map.Entry entry : storageMap.entrySet()) { storageTableNames.add(MetastoreUtil.getStorageTableName(fact, Storage.getPrefix((String) entry.getValue()))); } } else { storageTableNames.add(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName)); } return storageTableNames; }
public static Set<String> getDimensions(String name, Map<String, String> props) { Set<String> dimensions = new HashSet<>(); String dimStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeDimensionListKey(name)); dimensions.addAll(Arrays.asList(StringUtils.split(dimStr, ','))); return dimensions; }
public static Set<String> getMeasures(String name, Map<String, String> props) { Set<String> measures = new HashSet<>(); String measureStr = MetastoreUtil.getNamedStringValue(props, MetastoreUtil.getCubeMeasureListKey(name)); measures.addAll(Arrays.asList(StringUtils.split(measureStr, ','))); return measures; }
public void updateDimAttributeProperties() { MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeDimensionListKey(getName()), MetastoreUtil.getNamedSetFromStringSet(dimensions)); } public void updateMeasureProperties() {
public boolean isStorageTableCandidateForRange(String storageTableName, Date fromDate, Date toDate) throws LensException { List<Date> storageEndDates = getStorageTimes(storageTableName, MetastoreUtil.getStoragetableEndTimesKey()); for(Date endDate : storageEndDates) { // endDate is exclusive if (endDate.before(fromDate) || endDate.equals(fromDate)) { log.debug("from date {} is after validity end time: {}, hence discarding {}", fromDate, endDate, storageTableName); return false; } } List<Date> storageStartDates = getStorageTimes(storageTableName, MetastoreUtil.getStoragetableStartTimesKey()); for(Date startDate : storageStartDates) { // toDate is exclusive on the range if (startDate.after(toDate) || startDate.equals(toDate)) { log.debug("to date {} is before validity start time: {}, hence discarding {}", toDate, startDate, storageTableName); return false; } } return true; }
storageProps.put(getStoragetableStartTimesKey(), "now -30 days"); storageProps.put(getStoragetableEndTimesKey(), "now +10 days"); String storageTableName = getFactOrDimtableStorageTableName(factNameSkipPart, entry); assertTrue(client.tableExists(storageTableName));
Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c4))); table.getParameters().put(MetastoreUtil.getPartitionTimelineStorageClassKey(HOURLY, "ttd"), StoreAllPartitionTimeline.class.getCanonicalName()); table.getParameters().put(MetastoreUtil.getPartitionTimelineStorageClassKey(HOURLY, "ttd2"), StoreAllPartitionTimeline.class.getCanonicalName()); client.pushHiveTable(table); List<String> partitions = Lists.newArrayList(); StoreAllPartitionTimeline ttdStoreAll = new StoreAllPartitionTimeline(MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), c4), HOURLY, "ttd"); StoreAllPartitionTimeline ttd2StoreAll = new StoreAllPartitionTimeline(MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), c4), HOURLY, "ttd2"); while (!(temp.after(NOW))) { table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c4))); assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); assertTimeline(client, fact.getName(), c4, HOURLY, "ttd", ttdStoreAll); assertTimeline(client, fact.getName(), c4, HOURLY, "ttd2", ttd2StoreAll);
String storageTableName = MetastoreUtil.getStorageTableName(dimName, Storage.getPrefix(storage)); assertTrue(client.tableExists(storageTableName)); assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timeParts)); assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, getDatePartitionKey())); String storageTableName = getFactOrDimtableStorageTableName(dimName, c1); assertEquals(client.getAllParts(storageTableName).size(), 2); List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(1, parts.size()); assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName()); assertEquals(parts.get(0).getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 0));
public static String getFactOrDimtableStorageTableName(String factName, String storageName) { return getStorageTableName(factName, Storage.getPrefix(storageName)); }
private void assertTimeline(CubeMetastoreClient client, String factName, String storageName, UpdatePeriod updatePeriod, String timeDim, PartitionTimeline expectedTimeline) throws Exception { assertNotNull(factName); assertNotNull(storageName); assertNotNull(updatePeriod); assertNotNull(timeDim); String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName); List<PartitionTimeline> timelines = client.getTimelines(factName, storageName, updatePeriod.name(), timeDim); assertEquals(timelines.size(), 1); PartitionTimeline actualTimeline = timelines.get(0); assertEquals(actualTimeline, expectedTimeline); assertEquals(client.getTable(storageTableName).getParameters() .get(MetastoreUtil.getPartitionTimelineStorageClassKey(updatePeriod, timeDim)), expectedTimeline.getClass().getCanonicalName()); expectedTimeline.init(client.getTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName))); assertEquals(actualTimeline, expectedTimeline); }
String storageTableName = getFactOrDimtableStorageTableName(dimName, storage); assertTrue(client.tableExists(storageTableName)); String storageTableName = getFactOrDimtableStorageTableName(dimName, c1); assertFalse(client.dimTableLatestPartitionExists(storageTableName)); assertEquals(latestPart.getInputFormatClass(), TextInputFormat.class); assertFalse(latestPart.getCols().contains(newcol)); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 0)); assertEquals(latestPart.getInputFormatClass(), SequenceFileInputFormat.class); assertTrue(latestPart.getCols().contains(newcol)); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 1)); assertEquals(latestPart.getInputFormatClass(), TextInputFormat.class); assertTrue(latestPart.getCols().contains(newcol)); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 2)); latestPart = parts.get(0); assertEquals(latestPart.getInputFormatClass(), SequenceFileInputFormat.class); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 1)); assertEquals(client.getAllParts(storageTableName).size(), 3); latestPart = parts.get(0); assertEquals(latestPart.getInputFormatClass(), SequenceFileInputFormat.class); assertEquals(latestPart.getParameters().get(getLatestPartTimestampKey("dt")), getDateStringWithOffset(HOURLY, 1)); assertEquals(client.getAllParts(storageTableName).size(), 2);
/** * Return valid columns of the fact, which can be specified by property MetastoreUtil.getValidColumnsKey(getName()) * * @return */ public Set<String> getValidColumns() { String validColsStr = MetastoreUtil.getNamedStringValue(getProperties(), MetastoreUtil.getValidColumnsKey(getName())); return validColsStr == null ? null : new HashSet<>(Arrays.asList(StringUtils.split(validColsStr.toLowerCase(), ','))); }
public Set<String> getTimedDimensions() { String str = getProperties().get(MetastoreUtil.getCubeTimedDimensionListKey(getName())); Set<String> timedDimensions = new HashSet<>(); if (str != null) { timedDimensions.addAll(Arrays.asList(StringUtils.split(str, ','))); } return timedDimensions; }
@Override public void addProperties() { super.addProperties(); MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeMeasureListKey(getName()), measures); setMeasureProperties(getProperties(), measures); MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeDimensionListKey(getName()), dimensions); setDimensionProperties(getProperties(), dimensions); }
private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, String> nonTimeParts, Map<String, TreeSet<Date>> timePartSpecs, UpdatePeriod updatePeriod) throws HiveException, LensException { Table hiveTable = getHiveTable(storageTableName); String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS); if (timePartColsStr != null) { LatestInfo latest = new LatestInfo(); String[] timePartCols = StringUtils.split(timePartColsStr, ','); for (String partCol : timePartCols) { if (!timePartSpecs.containsKey(partCol)) { continue; } boolean makeLatest = true; Partition part = getLatestPart(storageTableName, partCol, nonTimeParts); Date pTimestamp = timePartSpecs.get(partCol).last(); Date latestTimestamp = getLatestTimeStampFromPartition(part, partCol); if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) { makeLatest = false; } if (makeLatest) { Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(partCol), updatePeriod.format(pTimestamp)); latest.latestParts.put(partCol, new LatestPartColumnInfo(latestParams)); } } return latest; } else { return null; } }
try { partitions = getClient().getPartitionsByFilter(hiveTable, StorageConstants.getPartFilter(nonTimePartSpec)); filterPartitionsByUpdatePeriod(partitions, updatePeriod); filterPartitionsByNonTimeParts(partitions, nonTimePartSpec, timeCol); } catch (TException e) { throw new HiveException(e); latest = new LatestInfo(); latest.setPart(nextLatest); Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(timeCol), nextLatest.getValues().get(timeColIndex)); latest.addLatestPartInfo(timeCol, new LatestPartColumnInfo(latestParams));
public void updateMeasureProperties() { MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeMeasureListKey(getName()), MetastoreUtil.getNamedSetFromStringSet(measures)); }
private void assertTestFactTimelineClass(CubeMetastoreClient client) throws Exception { String factName = "testFact"; client.getTimelines(factName, c1, null, null); client.getTimelines(factName, c4, null, null); client.clearHiveTableCache(); CubeFactTable fact = client.getCubeFactTable(factName); Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c1))); assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); for (UpdatePeriod period : Lists.newArrayList(MINUTELY, HOURLY, DAILY, MONTHLY, YEARLY, QUARTERLY)) { for (String partCol : Lists.newArrayList("dt")) { assertTimeline(client, fact.getName(), c1, period, partCol, EndsAndHolesPartitionTimeline.class); } } table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(), Storage.getPrefix(c4))); assertEquals(table.getParameters().get(MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true"); for (UpdatePeriod period : Lists.newArrayList(MINUTELY, HOURLY, DAILY, MONTHLY, YEARLY, QUARTERLY)) { for (String partCol : Lists.newArrayList("ttd", "ttd2")) { assertTimeline(client, fact.getName(), c4, period, partCol, EndsAndHolesPartitionTimeline.class); } } }