.get(factName, storageName).entrySet()) { if (updatePeriod == null || entry.getKey().equals(updatePeriod)) { for (Map.Entry<String, PartitionTimeline> entry1 : entry.getValue().entrySet()) {
private void loadTimelinesFromAllPartitions(String storageTableName, String timeLineKey) throws HiveException, LensException { // Then add all existing partitions for batch addition in respective timelines. Table storageTable = getTable(storageTableName); List<String> timeParts = getTimePartColNamesOfTable(storageTable); List<FieldSchema> partCols = storageTable.getPartCols(); for (Partition partition : getPartitionsByFilter(storageTableName, null)) { UpdatePeriod period = deduceUpdatePeriod(partition); List<String> values = partition.getValues(); if (values.contains(StorageConstants.LATEST_PARTITION_VALUE)) { log.info("dropping latest partition from fact storage table: {}. Spec: {}", storageTableName, partition.getSpec()); getClient().dropPartition(storageTableName, values, false); continue; } for (int i = 0; i < partCols.size(); i++) { if (timeParts.contains(partCols.get(i).getName())) { addForBatchAddition(timeLineKey, storageTableName, period, partCols.get(i).getName(), values.get(i)); } } } }
private void ensureEntryForTimeLineKey(String fact, String storage, UpdatePeriod updatePeriod, String storageTableName, String timeLineKey) throws LensException { // Not found in table properties either, compute from all partitions of the fact-storage table. // First make sure all combinations of update period and partition column have an entry even // if no partitions exist if (getFactTable(fact).getUpdatePeriods() != null && getFactTable(fact).getUpdatePeriods().get(storage) != null) { log.info("loading from all partitions: {}", storageTableName); Table storageTable = getTable(storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTable)) { ensureEntry(timeLineKey, storageTableName, updatePeriod, partCol); } } }
EndsAndHolesPartitionTimeline timelineDt = ((EndsAndHolesPartitionTimeline) client.partitionTimelineCache.get( factName, c1, HOURLY, getDatePartitionKey())); EndsAndHolesPartitionTimeline timelineIt = ((EndsAndHolesPartitionTimeline) client.partitionTimelineCache.get( factName, c1, HOURLY, itPart.getName())); EndsAndHolesPartitionTimeline timelineEt = ((EndsAndHolesPartitionTimeline) client.partitionTimelineCache.get( factName, c1, HOURLY, etPart.getName())); StoreAllPartitionTimeline timelineDtC2 = ((StoreAllPartitionTimeline) client.partitionTimelineCache.get( factName, c2, HOURLY, getDatePartitionKey())); StoreAllPartitionTimeline timelineItC2 = ((StoreAllPartitionTimeline) client.partitionTimelineCache.get( factName, c2, HOURLY, itPart.getName())); StoreAllPartitionTimeline timelineEtC2 = ((StoreAllPartitionTimeline) client.partitionTimelineCache.get( factName, c2, HOURLY, etPart.getName())); PartitionTimeline timeline1Temp = client.partitionTimelineCache.get(factName, c1, HOURLY, getDatePartitionKey()); PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, HOURLY, getDatePartitionKey());
.get(factName, c1, HOURLY, getDatePartitionKey())); StoreAllPartitionTimeline timelineItC1 = ((StoreAllPartitionTimeline) client.partitionTimelineCache .get(factName, c1, HOURLY, itPart.getName())); StoreAllPartitionTimeline timelineEtC1 = ((StoreAllPartitionTimeline) client.partitionTimelineCache .get(factName, c1, HOURLY, etPart.getName())); EndsAndHolesPartitionTimeline timelineDt = ((EndsAndHolesPartitionTimeline) client.partitionTimelineCache .get(factName, c2, HOURLY, getDatePartitionKey())); EndsAndHolesPartitionTimeline timelineIt = ((EndsAndHolesPartitionTimeline) client.partitionTimelineCache .get(factName, c2, HOURLY, itPart.getName())); EndsAndHolesPartitionTimeline timelineEt = ((EndsAndHolesPartitionTimeline) client.partitionTimelineCache .get(factName, c2, HOURLY, etPart.getName())); .get(factName, c1, HOURLY, getDatePartitionKey())); PartitionTimeline timeline1Temp = client.partitionTimelineCache.get(factName, c1, HOURLY, getDatePartitionKey()); PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, HOURLY, getDatePartitionKey());
if ("true".equalsIgnoreCase(storageTable.getParameters().get(getPartitionTimelineCachePresenceKey()))) { try { loadTimelinesFromTableProperties(updatePeriod, storageTableName, timeLineKey); } catch (Exception e) { ensureEntryForTimeLineKey(fact, storage, updatePeriod, storageTableName, timeLineKey); if (!uniqueStorageTables.contains(storageTableName)) { uniqueStorageTables.add(storageTableName); loadTimelinesFromAllPartitions(storageTableName, timeLineKey); ensureEntryForTimeLineKey(fact, storage, updatePeriod, storageTableName, timeLineKey); if (!uniqueStorageTables.contains(storageTableName)) { uniqueStorageTables.add(storageTableName); loadTimelinesFromAllPartitions(storageTableName, timeLineKey);
.get(factName, storageName).entrySet()) { if (updatePeriod == null || entry.getKey().equals(updatePeriod)) { for (Map.Entry<String, PartitionTimeline> entry1 : entry.getValue().entrySet()) {
for (String storage : fact.getStorages()) { for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) { PartitionTimeline timeline = partitionTimelineCache.get(fact.getSourceFactName(), storage, updatePeriod, partCol); if (timeline != null) {// this storage table is partitioned by partCol or not.
for (String storage : fact.getStorages()) { for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) { PartitionTimeline timeline = partitionTimelineCache.get(fact.getSourceFactName(), storage, updatePeriod, partCol); if (timeline != null) {// this storage table is partitioned by partCol or not.
if ("true".equalsIgnoreCase(storageTable.getParameters().get(getPartitionTimelineCachePresenceKey()))) { try { loadTimelinesFromTableProperties(updatePeriod, storageTableName, timeLineKey); } catch (Exception e) { ensureEntryForTimeLineKey(fact, storage, updatePeriod, storageTableName, timeLineKey); if (!uniqueStorageTables.contains(storageTableName)) { uniqueStorageTables.add(storageTableName); loadTimelinesFromAllPartitions(storageTableName, timeLineKey); ensureEntryForTimeLineKey(fact, storage, updatePeriod, storageTableName, timeLineKey); if (!uniqueStorageTables.contains(storageTableName)) { uniqueStorageTables.add(storageTableName); loadTimelinesFromAllPartitions(storageTableName, timeLineKey);
/** * get all timelines for all update periods and partition columns for the given fact-storage pair. If already loaded * in memory, it'll return that. If not, it'll first try to load it from table properties. If not found in table * properties, it'll get all partitions, compute timelines in memory, write back all loads timelines to table * properties for further usage and return them. * * @param fact fact * @param storage storage * @return all timelines for fact-storage pair. Load from properties/all partitions if needed. * @throws HiveException * @throws LensException */ public TreeMap<UpdatePeriod, CaseInsensitiveStringHashMap<PartitionTimeline>> get(String fact, String storage) throws HiveException, LensException { // SUSPEND CHECKSTYLE CHECK DoubleCheckedLockingCheck // Unique key for the timeline cache, based on storage and fact. String timeLineKey = (Storage.getPrefix(storage)+ fact).toLowerCase(); synchronized (this) { if (get(timeLineKey) == null) { loadTimeLines(fact, storage, timeLineKey); } log.debug("timeline for {} is: {}", storage, get(timeLineKey)); // return the final value from memory return get(timeLineKey); // RESUME CHECKSTYLE CHECK DoubleCheckedLockingCheck } }
/** * get all timelines for all update periods and partition columns for the given fact-storage pair. If already loaded * in memory, it'll return that. If not, it'll first try to load it from table properties. If not found in table * properties, it'll get all partitions, compute timelines in memory, write back all loads timelines to table * properties for further usage and return them. * * @param fact fact * @param storage storage * @return all timelines for fact-storage pair. Load from properties/all partitions if needed. * @throws HiveException * @throws LensException */ public TreeMap<UpdatePeriod, CaseInsensitiveStringHashMap<PartitionTimeline>> get(String fact, String storage) throws HiveException, LensException { // SUSPEND CHECKSTYLE CHECK DoubleCheckedLockingCheck // Unique key for the timeline cache, based on storage and fact. String timeLineKey = (Storage.getPrefix(storage)+ fact).toLowerCase(); synchronized (this) { if (get(timeLineKey) == null) { loadTimeLines(fact, storage, timeLineKey); } log.debug("timeline for {} is: {}", storage, get(timeLineKey)); // return the final value from memory return get(timeLineKey); // RESUME CHECKSTYLE CHECK DoubleCheckedLockingCheck } }
/** * * @param fact fact * @param storage storage * @param partCol part column * @return true if all the timelines for fact-storage table are empty for all valid update periods. * @throws HiveException * @throws LensException */ public boolean noPartitionsExist(String fact, String storage, String partCol) throws HiveException, LensException { if (get(fact, storage) == null) { return true; } for (UpdatePeriod updatePeriod : get(fact, storage).keySet()) { PartitionTimeline timeline = get(fact, storage, updatePeriod, partCol); if (timeline != null && !timeline.isEmpty()) { return false; } } return true; }
/** * * @param fact fact * @param storage storage * @param partCol part column * @return true if all the timelines for fact-storage table are empty for all valid update periods. * @throws HiveException * @throws LensException */ public boolean noPartitionsExist(String fact, String storage, String partCol) throws HiveException, LensException { if (get(fact, storage) == null) { return true; } for (UpdatePeriod updatePeriod : get(fact, storage).keySet()) { PartitionTimeline timeline = get(fact, storage, updatePeriod, partCol); if (timeline != null && !timeline.isEmpty()) { return false; } } return true; }
private void loadTimelinesFromAllPartitions(String storageTableName, String timeLineKey) throws HiveException, LensException { // Then add all existing partitions for batch addition in respective timelines. Table storageTable = getTable(storageTableName); List<String> timeParts = getTimePartColNamesOfTable(storageTable); List<FieldSchema> partCols = storageTable.getPartCols(); for (Partition partition : getPartitionsByFilter(storageTableName, null)) { UpdatePeriod period = deduceUpdatePeriod(partition); List<String> values = partition.getValues(); if (values.contains(StorageConstants.LATEST_PARTITION_VALUE)) { log.info("dropping latest partition from fact storage table: {}. Spec: {}", storageTableName, partition.getSpec()); getClient().dropPartition(storageTableName, values, false); continue; } for (int i = 0; i < partCols.size(); i++) { if (timeParts.contains(partCols.get(i).getName())) { addForBatchAddition(timeLineKey, storageTableName, period, partCols.get(i).getName(), values.get(i)); } } } }
/** * Adds given partition(for storageTable, updatePeriod, partitionColum=partition) for batch addition in an * appropriate timeline object. Ignore if partition is not valid. * * @param timeLineKey key for the timeLine map * @param storageTableName hive table name * @param updatePeriod update period * @param partitionColumn partition column * @param partition partition */ public void addForBatchAddition(String timeLineKey, String storageTableName, UpdatePeriod updatePeriod, String partitionColumn, String partition) { try { ensureEntry(timeLineKey, storageTableName, updatePeriod, partitionColumn) .addForBatchAddition(TimePartition.of(updatePeriod, partition)); } catch (LensException e) { // to take care of the case where partition name is something like `latest` log.error("Couldn't parse partition: {} with update period: {}, skipping.", partition, updatePeriod, e); } }
/** * store back all timelines of given storage to table properties * * @param timeLineKey key for the time line * @param storageTableName Storage table name * @throws HiveException */ private void alterTablePartitionCache(String timeLineKey, UpdatePeriod updatePeriod, String storageTableName) throws HiveException, LensException { Table table = getTable(storageTableName); Map<String, String> params = table.getParameters(); if (partitionTimelineCache.get(timeLineKey) != null) { for (Map.Entry<String, PartitionTimeline> entry : partitionTimelineCache.get(timeLineKey).get(updatePeriod) .entrySet()) { entry.getValue().updateTableParams(table); } params.put(getPartitionTimelineCachePresenceKey(), "true"); alterHiveTable(storageTableName, table); } }
/** * Adds given partition(for storageTable, updatePeriod, partitionColum=partition) for batch addition in an * appropriate timeline object. Ignore if partition is not valid. * * @param timeLineKey key for the timeLine map * @param storageTableName hive table name * @param updatePeriod update period * @param partitionColumn partition column * @param partition partition */ public void addForBatchAddition(String timeLineKey, String storageTableName, UpdatePeriod updatePeriod, String partitionColumn, String partition) { try { ensureEntry(timeLineKey, storageTableName, updatePeriod, partitionColumn) .addForBatchAddition(TimePartition.of(updatePeriod, partition)); } catch (LensException e) { // to take care of the case where partition name is something like `latest` log.error("Couldn't parse partition: {} with update period: {}, skipping.", partition, updatePeriod, e); } }
/** * store back all timelines of given storage to table properties * * @param timeLineKey key for the time line * @param storageTableName Storage table name * @throws HiveException */ private void alterTablePartitionCache(String timeLineKey, UpdatePeriod updatePeriod, String storageTableName) throws HiveException, LensException { Table table = getTable(storageTableName); Map<String, String> params = table.getParameters(); if (partitionTimelineCache.get(timeLineKey) != null) { for (Map.Entry<String, PartitionTimeline> entry : partitionTimelineCache.get(timeLineKey).get(updatePeriod) .entrySet()) { entry.getValue().updateTableParams(table); } params.put(getPartitionTimelineCachePresenceKey(), "true"); alterHiveTable(storageTableName, table); } }
/** update partition timeline cache for deletion of time partition */ public boolean updateForDeletion(String cubeTableName, String storageName, UpdatePeriod updatePeriod, Map<String, Date> timePartSpec) throws HiveException, LensException { // fail fast. All part cols mentioned in all partitions should exist. for (String partCol : timePartSpec.keySet()) { getAndFailFast(cubeTableName, storageName, updatePeriod, partCol); } boolean updated = false; for (Map.Entry<String, Date> entry : timePartSpec.entrySet()) { TimePartition part = TimePartition.of(updatePeriod, entry.getValue()); if (!partitionExistsByFilter(cubeTableName, storageName, updatePeriod, StorageConstants.getPartFilter(entry.getKey(), part.getDateString()))) { get(cubeTableName, storageName, updatePeriod, entry.getKey()).drop(part); updated = true; } } return updated; } }