private void ensureEntryForTimeLineKey(String fact, String storage, UpdatePeriod updatePeriod, String storageTableName, String timeLineKey) throws LensException { // Not found in table properties either, compute from all partitions of the fact-storage table. // First make sure all combinations of update period and partition column have an entry even // if no partitions exist if (getFactTable(fact).getUpdatePeriods() != null && getFactTable(fact).getUpdatePeriods().get(storage) != null) { log.info("loading from all partitions: {}", storageTableName); Table storageTable = getTable(storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTable)) { ensureEntry(timeLineKey, storageTableName, updatePeriod, partCol); } } }
private void ensureEntryForTimeLineKey(String fact, String storage, UpdatePeriod updatePeriod, String storageTableName, String timeLineKey) throws LensException { // Not found in table properties either, compute from all partitions of the fact-storage table. // First make sure all combinations of update period and partition column have an entry even // if no partitions exist if (getFactTable(fact).getUpdatePeriods() != null && getFactTable(fact).getUpdatePeriods().get(storage) != null) { log.info("loading from all partitions: {}", storageTableName); Table storageTable = getTable(storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTable)) { ensureEntry(timeLineKey, storageTableName, updatePeriod, partCol); } } }
private void loadTimelinesFromTableProperties(UpdatePeriod updatePeriod, String storageTableName, String timeLineKey) throws HiveException, LensException { log.info("loading from table properties: {}", storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTableName)) { ensureEntry(timeLineKey, storageTableName, updatePeriod, partCol).init(getTable(storageTableName)); } }
/** * Adds given partition(for storageTable, updatePeriod, partitionColum=partition) for batch addition in an * appropriate timeline object. Ignore if partition is not valid. * * @param timeLineKey key for the timeLine map * @param storageTableName hive table name * @param updatePeriod update period * @param partitionColumn partition column * @param partition partition */ public void addForBatchAddition(String timeLineKey, String storageTableName, UpdatePeriod updatePeriod, String partitionColumn, String partition) { try { ensureEntry(timeLineKey, storageTableName, updatePeriod, partitionColumn) .addForBatchAddition(TimePartition.of(updatePeriod, partition)); } catch (LensException e) { // to take care of the case where partition name is something like `latest` log.error("Couldn't parse partition: {} with update period: {}, skipping.", partition, updatePeriod, e); } }
/** * Adds given partition(for storageTable, updatePeriod, partitionColum=partition) for batch addition in an * appropriate timeline object. Ignore if partition is not valid. * * @param timeLineKey key for the timeLine map * @param storageTableName hive table name * @param updatePeriod update period * @param partitionColumn partition column * @param partition partition */ public void addForBatchAddition(String timeLineKey, String storageTableName, UpdatePeriod updatePeriod, String partitionColumn, String partition) { try { ensureEntry(timeLineKey, storageTableName, updatePeriod, partitionColumn) .addForBatchAddition(TimePartition.of(updatePeriod, partition)); } catch (LensException e) { // to take care of the case where partition name is something like `latest` log.error("Couldn't parse partition: {} with update period: {}, skipping.", partition, updatePeriod, e); } }
private void loadTimelinesFromTableProperties(UpdatePeriod updatePeriod, String storageTableName, String timeLineKey) throws HiveException, LensException { log.info("loading from table properties: {}", storageTableName); for (String partCol : getTimePartColNamesOfTable(storageTableName)) { ensureEntry(timeLineKey, storageTableName, updatePeriod, partCol).init(getTable(storageTableName)); } }