private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, String> nonTimeParts, Map<String, TreeSet<Date>> timePartSpecs, UpdatePeriod updatePeriod) throws HiveException, LensException { Table hiveTable = getHiveTable(storageTableName); String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS); if (timePartColsStr != null) { LatestInfo latest = new LatestInfo(); String[] timePartCols = StringUtils.split(timePartColsStr, ','); for (String partCol : timePartCols) { if (!timePartSpecs.containsKey(partCol)) { continue; } boolean makeLatest = true; Partition part = getLatestPart(storageTableName, partCol, nonTimeParts); Date pTimestamp = timePartSpecs.get(partCol).last(); Date latestTimestamp = getLatestTimeStampFromPartition(part, partCol); if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) { makeLatest = false; } if (makeLatest) { Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(partCol), updatePeriod.format(pTimestamp)); latest.latestParts.put(partCol, new LatestPartColumnInfo(latestParams)); } } return latest; } else { return null; } }
private LatestInfo getDimTableLatestInfo(String storageTableName, Map<String, String> nonTimeParts, Map<String, TreeSet<Date>> timePartSpecs, UpdatePeriod updatePeriod) throws HiveException, LensException { Table hiveTable = getHiveTable(storageTableName); String timePartColsStr = hiveTable.getTTable().getParameters().get(MetastoreConstants.TIME_PART_COLUMNS); if (timePartColsStr != null) { LatestInfo latest = new LatestInfo(); String[] timePartCols = StringUtils.split(timePartColsStr, ','); for (String partCol : timePartCols) { if (!timePartSpecs.containsKey(partCol)) { continue; } boolean makeLatest = true; Partition part = getLatestPart(storageTableName, partCol, nonTimeParts); Date pTimestamp = timePartSpecs.get(partCol).last(); Date latestTimestamp = getLatestTimeStampFromPartition(part, partCol); if (latestTimestamp != null && pTimestamp.before(latestTimestamp)) { makeLatest = false; } if (makeLatest) { Map<String, String> latestParams = LensUtil.getHashMap(getLatestPartTimestampKey(partCol), updatePeriod.format(pTimestamp)); latest.latestParts.put(partCol, new LatestPartColumnInfo(latestParams)); } } return latest; } else { return null; } }
private void assertLatestForRegions(String storageTableName, Map<String, Date> expectedLatestValues) throws HiveException, LensException { List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'"); assertEquals(parts.size(), expectedLatestValues.size()); for (Partition part : parts) { assertEquals(MetastoreUtil.getLatestTimeStampFromPartition(part, getDatePartitionKey()), TimePartition.of(HOURLY, expectedLatestValues.get(part.getSpec().get("region"))).getDate()); } }
Partition part = getLatestPart(storageTableName, timeCol, nonTimePartSpec); Date latestTimestamp = getLatestTimeStampFromPartition(part, timeCol); Date dropTimestamp; try {
Partition part = getLatestPart(storageTableName, timeCol, nonTimePartSpec); Date latestTimestamp = getLatestTimeStampFromPartition(part, timeCol); Date dropTimestamp; try {