public String value() { return name(); }
public static XUpdatePeriod fromValue(String v) { return valueOf(v); }
xp.getPartitionParameters().getProperty().addAll(xPropertiesFromMap(p.getParameters())); String upParam = p.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD); xp.setUpdatePeriod(XUpdatePeriod.valueOf(upParam)); LinkedHashMap<String, String> partSpec = p.getSpec(); xp.setFullPartitionSpec(new XPartSpec()); timePartSpecElement.setKey(entry.getKey()); timePartSpecElement .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).parse( entry.getValue()))); xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpecElement);
@Override public void updatePartition(LensSessionHandle sessionid, String tblName, String storageName, XPartition xPartition) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient client = getClient(sessionid); String storageTableName = client .getStorageTableName(tblName, storageName, UpdatePeriod.valueOf(xPartition.getUpdatePeriod().name())); Partition existingPartition = client.getPartitionByFilter(storageTableName, StorageConstants.getPartFilter(JAXBUtils.getFullPartSpecAsMap(xPartition))); JAXBUtils.updatePartitionFromXPartition(existingPartition, xPartition); client.updatePartition(tblName, storageName, existingPartition, UpdatePeriod.valueOf(xPartition.getUpdatePeriod().value())); } catch (HiveException | ClassNotFoundException | InvalidOperationException | UnsupportedOperationException exc) { throw new LensException(exc); } }
public static Map<String, Map<UpdatePeriod, String>> storageTablePrefixMapOfStorage(XStorageTables storageTables) { Map<String, Map<UpdatePeriod, String>> storageTableMap = new HashMap<>(); if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { for (XStorageTableElement sTbl : storageTables.getStorageTable()) { Map<UpdatePeriod, String> storageNameMap = new HashMap<>(); if (sTbl.getUpdatePeriods() != null && sTbl.getUpdatePeriods().getUpdatePeriodTableDescriptor() != null && !sTbl .getUpdatePeriods().getUpdatePeriodTableDescriptor().isEmpty()) { for (XUpdatePeriodTableDescriptor updatePeriodTable : sTbl.getUpdatePeriods() .getUpdatePeriodTableDescriptor()) { // Get table name with update period as the prefix. storageNameMap.put(UpdatePeriod.valueOf(updatePeriodTable.getUpdatePeriod().value()), updatePeriodTable.getUpdatePeriod() + "_" + sTbl.getStorageName()); } } else { for (XUpdatePeriod updatePeriod : sTbl.getUpdatePeriods().getUpdatePeriod()) { storageNameMap.put(UpdatePeriod.valueOf(updatePeriod.value()), sTbl.getStorageName()); } } storageTableMap.put(sTbl.getStorageName(), storageNameMap); } } return storageTableMap; }
xp.getPartitionParameters().getProperty().addAll(xPropertiesFromMap(p.getParameters())); String upParam = p.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD); xp.setUpdatePeriod(XUpdatePeriod.valueOf(upParam)); LinkedHashMap<String, String> partSpec = p.getSpec(); xp.setFullPartitionSpec(new XPartSpec()); timePartSpecElement.setKey(entry.getKey()); timePartSpecElement .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).parse( entry.getValue()))); xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpecElement);
public static Map<String, Map<UpdatePeriod, String>> storageTablePrefixMapOfStorage(XStorageTables storageTables) { Map<String, Map<UpdatePeriod, String>> storageTableMap = new HashMap<>(); if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { for (XStorageTableElement sTbl : storageTables.getStorageTable()) { Map<UpdatePeriod, String> storageNameMap = new HashMap<>(); if (sTbl.getUpdatePeriods() != null && sTbl.getUpdatePeriods().getUpdatePeriodTableDescriptor() != null && !sTbl .getUpdatePeriods().getUpdatePeriodTableDescriptor().isEmpty()) { for (XUpdatePeriodTableDescriptor updatePeriodTable : sTbl.getUpdatePeriods() .getUpdatePeriodTableDescriptor()) { // Get table name with update period as the prefix. storageNameMap.put(UpdatePeriod.valueOf(updatePeriodTable.getUpdatePeriod().value()), updatePeriodTable.getUpdatePeriod() + "_" + sTbl.getStorageName()); } } else { for (XUpdatePeriod updatePeriod : sTbl.getUpdatePeriods().getUpdatePeriod()) { storageNameMap.put(UpdatePeriod.valueOf(updatePeriod.value()), sTbl.getStorageName()); } } storageTableMap.put(sTbl.getStorageName(), storageNameMap); } } return storageTableMap; }
public String value() { return name(); }
public static XUpdatePeriod fromValue(String v) { return valueOf(v); }
@Override public void updatePartitions(LensSessionHandle sessionid, String tblName, String storageName, XPartitionList xPartitions) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient client = getClient(sessionid); Set<String> storageTableNames = getAllTablesForStorage(sessionid, tblName, storageName); Map<UpdatePeriod, List<Partition>> partitionsToUpdate = new HashMap<>(); for (String storageTableName : storageTableNames) { for (XPartition xPartition : xPartitions.getPartition()) { Partition existingPartition = client.getPartitionByFilter(storageTableName, StorageConstants.getPartFilter(JAXBUtils.getFullPartSpecAsMap(xPartition))); JAXBUtils.updatePartitionFromXPartition(existingPartition, xPartition); UpdatePeriod updatePeriod = UpdatePeriod.valueOf(xPartition.getUpdatePeriod().value()); List<Partition> partitionList = partitionsToUpdate.get(updatePeriod); if (partitionList == null) { partitionList = new ArrayList<>(); partitionsToUpdate.put(updatePeriod, partitionList); } partitionList.add(existingPartition); } } client.updatePartitions(tblName, storageName, partitionsToUpdate); } catch (HiveException | ClassNotFoundException | InvalidOperationException exc) { throw new LensException(exc); } }
public static Map<String, UpdatePeriod> dumpPeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, UpdatePeriod> dumpPeriods = new LinkedHashMap<String, UpdatePeriod>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { UpdatePeriod dumpPeriod = null; if (ste.getUpdatePeriods() != null && !ste.getUpdatePeriods().getUpdatePeriod().isEmpty()) { dumpPeriod = UpdatePeriod.valueOf(ste.getUpdatePeriods().getUpdatePeriod().get(0).name()); } dumpPeriods.put(ste.getStorageName(), dumpPeriod); } return dumpPeriods; } return null; }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; }
public static Map<String, UpdatePeriod> dumpPeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, UpdatePeriod> dumpPeriods = new LinkedHashMap<String, UpdatePeriod>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { UpdatePeriod dumpPeriod = null; if (ste.getUpdatePeriods() != null && !ste.getUpdatePeriods().getUpdatePeriod().isEmpty()) { dumpPeriod = UpdatePeriod.valueOf(ste.getUpdatePeriods().getUpdatePeriod().get(0).name()); } dumpPeriods.put(ste.getStorageName(), dumpPeriod); } return dumpPeriods; } return null; }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; } /**
public static Map<String, Set<UpdatePeriod>> getFactUpdatePeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, Set<UpdatePeriod>> factUpdatePeriods = new LinkedHashMap<String, Set<UpdatePeriod>>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { Set<UpdatePeriod> updatePeriods = new TreeSet<>(); // Check if the update period array is empty. List<XUpdatePeriod> updatePeriodList = ste.getUpdatePeriods().getUpdatePeriod(); if (updatePeriodList.isEmpty()) { List<XUpdatePeriodTableDescriptor> tableDescriptorList = ste.getUpdatePeriods() .getUpdatePeriodTableDescriptor(); for (XUpdatePeriodTableDescriptor tableDescriptor : tableDescriptorList) { updatePeriodList.add(tableDescriptor.getUpdatePeriod()); } } for (XUpdatePeriod upd : updatePeriodList) { updatePeriods.add(UpdatePeriod.valueOf(upd.name())); } factUpdatePeriods.put(ste.getStorageName(), updatePeriods); } return factUpdatePeriods; } return null; }
public XStorageTableElement getStorageOfFact(LensSessionHandle sessionid, String fact, String storageName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); FactTable factTable = msClient.getFactTable(fact); Set<UpdatePeriod> updatePeriods = factTable.getUpdatePeriods().get(storageName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName))); tblElement.setStorageName(storageName); for (UpdatePeriod p : updatePeriods) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } return tblElement; } }
public static Map<String, Set<UpdatePeriod>> getFactUpdatePeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, Set<UpdatePeriod>> factUpdatePeriods = new LinkedHashMap<String, Set<UpdatePeriod>>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { Set<UpdatePeriod> updatePeriods = new TreeSet<>(); // Check if the update period array is empty. List<XUpdatePeriod> updatePeriodList = ste.getUpdatePeriods().getUpdatePeriod(); if (updatePeriodList.isEmpty()) { List<XUpdatePeriodTableDescriptor> tableDescriptorList = ste.getUpdatePeriods() .getUpdatePeriodTableDescriptor(); for (XUpdatePeriodTableDescriptor tableDescriptor : tableDescriptorList) { updatePeriodList.add(tableDescriptor.getUpdatePeriod()); } } for (XUpdatePeriod upd : updatePeriodList) { updatePeriods.add(UpdatePeriod.valueOf(upd.name())); } factUpdatePeriods.put(ste.getStorageName(), updatePeriods); } return factUpdatePeriods; } return null; }
public XStorageTableElement getStorageOfDim(LensSessionHandle sessionid, String dimTblName, String storageName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } return tblElement; } }
@Override public void addStorageToFact(LensSessionHandle sessionid, String fact, XStorageTableElement storageTable) throws LensException { Set<UpdatePeriod> updatePeriods = new TreeSet<>(); for (XUpdatePeriod sup : storageTable.getUpdatePeriods().getUpdatePeriod()) { updatePeriods.add(UpdatePeriod.valueOf(sup.name())); } try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); XStorageTables tables = new XStorageTables(); tables.getStorageTable().add(storageTable); msClient.addStorage(msClient.getCubeFactTable(fact), storageTable.getStorageName(), updatePeriods, JAXBUtils.tableDescPrefixMapFromXStorageTables(tables), JAXBUtils.storageTablePrefixMapOfStorage(tables).get(storageTable.getStorageName())); log.info("Added storage " + storageTable.getStorageName() + ":" + updatePeriods + " for fact " + fact); } }
tblElement.setStorageName(storageName); for (UpdatePeriod p : updatePeriods) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), (String) entry.getValue())))); updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name())); xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);