public String value() { return name(); }
public String value() { return name(); }
public static Map<String, UpdatePeriod> dumpPeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, UpdatePeriod> dumpPeriods = new LinkedHashMap<String, UpdatePeriod>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { UpdatePeriod dumpPeriod = null; if (ste.getUpdatePeriods() != null && !ste.getUpdatePeriods().getUpdatePeriod().isEmpty()) { dumpPeriod = UpdatePeriod.valueOf(ste.getUpdatePeriods().getUpdatePeriod().get(0).name()); } dumpPeriods.put(ste.getStorageName(), dumpPeriod); } return dumpPeriods; } return null; }
public static Map<String, UpdatePeriod> dumpPeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, UpdatePeriod> dumpPeriods = new LinkedHashMap<String, UpdatePeriod>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { UpdatePeriod dumpPeriod = null; if (ste.getUpdatePeriods() != null && !ste.getUpdatePeriods().getUpdatePeriod().isEmpty()) { dumpPeriod = UpdatePeriod.valueOf(ste.getUpdatePeriods().getUpdatePeriod().get(0).name()); } dumpPeriods.put(ste.getStorageName(), dumpPeriod); } return dumpPeriods; } return null; }
public static Map<String, Set<UpdatePeriod>> getFactUpdatePeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, Set<UpdatePeriod>> factUpdatePeriods = new LinkedHashMap<String, Set<UpdatePeriod>>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { Set<UpdatePeriod> updatePeriods = new TreeSet<>(); // Check if the update period array is empty. List<XUpdatePeriod> updatePeriodList = ste.getUpdatePeriods().getUpdatePeriod(); if (updatePeriodList.isEmpty()) { List<XUpdatePeriodTableDescriptor> tableDescriptorList = ste.getUpdatePeriods() .getUpdatePeriodTableDescriptor(); for (XUpdatePeriodTableDescriptor tableDescriptor : tableDescriptorList) { updatePeriodList.add(tableDescriptor.getUpdatePeriod()); } } for (XUpdatePeriod upd : updatePeriodList) { updatePeriods.add(UpdatePeriod.valueOf(upd.name())); } factUpdatePeriods.put(ste.getStorageName(), updatePeriods); } return factUpdatePeriods; } return null; }
public static Map<String, Set<UpdatePeriod>> getFactUpdatePeriodsFromStorageTables(XStorageTables storageTables) { if (storageTables != null && !storageTables.getStorageTable().isEmpty()) { Map<String, Set<UpdatePeriod>> factUpdatePeriods = new LinkedHashMap<String, Set<UpdatePeriod>>(); for (XStorageTableElement ste : storageTables.getStorageTable()) { Set<UpdatePeriod> updatePeriods = new TreeSet<>(); // Check if the update period array is empty. List<XUpdatePeriod> updatePeriodList = ste.getUpdatePeriods().getUpdatePeriod(); if (updatePeriodList.isEmpty()) { List<XUpdatePeriodTableDescriptor> tableDescriptorList = ste.getUpdatePeriods() .getUpdatePeriodTableDescriptor(); for (XUpdatePeriodTableDescriptor tableDescriptor : tableDescriptorList) { updatePeriodList.add(tableDescriptor.getUpdatePeriod()); } } for (XUpdatePeriod upd : updatePeriodList) { updatePeriods.add(UpdatePeriod.valueOf(upd.name())); } factUpdatePeriods.put(ste.getStorageName(), updatePeriods); } return factUpdatePeriods; } return null; }
public static Map<String, String> getFullPartSpecAsMap(XPartition partition) { Map<String, String> spec = Maps.newHashMap(); if (partition.getTimePartitionSpec() != null) { for (XTimePartSpecElement timePartSpecElement : partition.getTimePartitionSpec().getPartSpecElement()) { spec.put(timePartSpecElement.getKey(), UpdatePeriod.valueOf(partition.getUpdatePeriod().name()).format() .format(getDateFromXML(timePartSpecElement.getValue()))); } } if (partition.getNonTimePartitionSpec() != null) { for (XPartSpecElement partSpecElement : partition.getNonTimePartitionSpec().getPartSpecElement()) { spec.put(partSpecElement.getKey(), partSpecElement.getValue()); } } return spec; } }
@Override public void addStorageToFact(LensSessionHandle sessionid, String fact, XStorageTableElement storageTable) throws LensException { Set<UpdatePeriod> updatePeriods = new TreeSet<>(); for (XUpdatePeriod sup : storageTable.getUpdatePeriods().getUpdatePeriod()) { updatePeriods.add(UpdatePeriod.valueOf(sup.name())); } try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); XStorageTables tables = new XStorageTables(); tables.getStorageTable().add(storageTable); msClient.addStorage(msClient.getCubeFactTable(fact), storageTable.getStorageName(), updatePeriods, JAXBUtils.tableDescPrefixMapFromXStorageTables(tables), JAXBUtils.storageTablePrefixMapOfStorage(tables).get(storageTable.getStorageName())); log.info("Added storage " + storageTable.getStorageName() + ":" + updatePeriods + " for fact " + fact); } }
public static Map<String, String> getFullPartSpecAsMap(XPartition partition) { Map<String, String> spec = Maps.newHashMap(); if (partition.getTimePartitionSpec() != null) { for (XTimePartSpecElement timePartSpecElement : partition.getTimePartitionSpec().getPartSpecElement()) { spec.put(timePartSpecElement.getKey(), UpdatePeriod.valueOf(partition.getUpdatePeriod().name()).format() .format(getDateFromXML(timePartSpecElement.getValue()))); } } if (partition.getNonTimePartitionSpec() != null) { for (XPartSpecElement partSpecElement : partition.getNonTimePartitionSpec().getPartSpecElement()) { spec.put(partSpecElement.getKey(), partSpecElement.getValue()); } } return spec; } }
public static void updatePartitionFromXPartition(Partition partition, XPartition xp) throws ClassNotFoundException { partition.getParameters().putAll(mapFromXProperties(xp.getPartitionParameters())); partition.getTPartition().getSd().getSerdeInfo().setParameters(mapFromXProperties(xp.getSerdeParameters())); partition.setLocation(xp.getLocation()); if (xp.getInputFormat() != null) { partition.setInputFormatClass(Class.forName(xp.getInputFormat()).asSubclass(InputFormat.class)); } if (xp.getOutputFormat() != null) { Class<? extends HiveOutputFormat> outputFormatClass = Class.forName(xp.getOutputFormat()).asSubclass(HiveOutputFormat.class); partition.setOutputFormatClass(outputFormatClass); } partition.getParameters().put(MetastoreConstants.PARTITION_UPDATE_PERIOD, xp.getUpdatePeriod().name()); partition.getTPartition().getSd().getSerdeInfo().setSerializationLib(xp.getSerdeClassname()); }
@Override public void addDimTableStorage(LensSessionHandle sessionid, String dimTblName, XStorageTableElement storageTable) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); UpdatePeriod period = null; if (storageTable.getUpdatePeriods() != null && !storageTable.getUpdatePeriods().getUpdatePeriod().isEmpty()) { period = UpdatePeriod.valueOf(storageTable.getUpdatePeriods().getUpdatePeriod().get(0).name()); } msClient.addStorage(dimTable, storageTable.getStorageName(), period, JAXBUtils.storageTableDescFromXStorageTableDesc(storageTable.getTableDesc())); log.info("Added storage " + storageTable.getStorageName() + " for dimension table " + dimTblName + " with update period " + period); } }
public static void updatePartitionFromXPartition(Partition partition, XPartition xp) throws ClassNotFoundException { partition.getParameters().putAll(mapFromXProperties(xp.getPartitionParameters())); partition.getTPartition().getSd().getSerdeInfo().setParameters(mapFromXProperties(xp.getSerdeParameters())); partition.setLocation(xp.getLocation()); if (xp.getInputFormat() != null) { partition.setInputFormatClass(Class.forName(xp.getInputFormat()).asSubclass(InputFormat.class)); } if (xp.getOutputFormat() != null) { Class<? extends HiveOutputFormat> outputFormatClass = Class.forName(xp.getOutputFormat()).asSubclass(HiveOutputFormat.class); partition.setOutputFormatClass(outputFormatClass); } partition.getParameters().put(MetastoreConstants.PARTITION_UPDATE_PERIOD, xp.getUpdatePeriod().name()); partition.getTPartition().getSd().getSerdeInfo().setSerializationLib(xp.getSerdeClassname()); }
timePartSpecElement.setKey(entry.getKey()); timePartSpecElement .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).parse( entry.getValue()))); xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpecElement);
timePartSpecElement.setKey(entry.getKey()); timePartSpecElement .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).parse( entry.getValue()))); xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpecElement);
assertEquals(timePartSpec.getKey(), fullPartSpec.getKey()); assertEquals( UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format(JAXBUtils.getDateFromXML(timePartSpec.getValue())), fullPartSpec.getValue()); DateTime date = target().path("metastore/cubes").path("testCube").path("latestdate")
XPartSpecElement fullPartSpec = postedPartition.getFullPartitionSpec().getPartSpecElement().iterator().next(); assertEquals(timePartSpec.getKey(), fullPartSpec.getKey()); assertEquals(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format(JAXBUtils.getDateFromXML( timePartSpec.getValue())), fullPartSpec.getValue());
XPartSpecElement fullPartSpec = readPartition.getFullPartitionSpec().getPartSpecElement().iterator().next(); assertEquals(timePartSpec.getKey(), fullPartSpec.getKey()); assertEquals(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format(JAXBUtils.getDateFromXML( timePartSpec.getValue())), fullPartSpec.getValue()); DateTime date =
@Override public void updatePartition(LensSessionHandle sessionid, String tblName, String storageName, XPartition xPartition) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient client = getClient(sessionid); String storageTableName = client .getStorageTableName(tblName, storageName, UpdatePeriod.valueOf(xPartition.getUpdatePeriod().name())); Partition existingPartition = client.getPartitionByFilter(storageTableName, StorageConstants.getPartFilter(JAXBUtils.getFullPartSpecAsMap(xPartition))); JAXBUtils.updatePartitionFromXPartition(existingPartition, xPartition); client.updatePartition(tblName, storageName, existingPartition, UpdatePeriod.valueOf(xPartition.getUpdatePeriod().value())); } catch (HiveException | ClassNotFoundException | InvalidOperationException | UnsupportedOperationException exc) { throw new LensException(exc); } }
public static StoragePartitionDesc storagePartSpecFromXPartition( XPartition xpart) { StoragePartitionDesc partDesc = new StoragePartitionDesc( xpart.getFactOrDimensionTableName(), timePartSpecfromXTimePartSpec(xpart.getTimePartitionSpec()), nonTimePartSpecfromXNonTimePartSpec(xpart.getNonTimePartitionSpec()), UpdatePeriod.valueOf(xpart.getUpdatePeriod().name())); partDesc.setPartParams(mapFromXProperties(xpart.getPartitionParameters())); partDesc.setSerdeParams(mapFromXProperties(xpart.getSerdeParameters())); partDesc.setLocation(xpart.getLocation()); partDesc.setInputFormat(xpart.getInputFormat()); partDesc.setOutputFormat(xpart.getOutputFormat()); partDesc.setSerializationLib(xpart.getSerdeClassname()); return partDesc; }
public static StoragePartitionDesc storagePartSpecFromXPartition( XPartition xpart) { StoragePartitionDesc partDesc = new StoragePartitionDesc( xpart.getFactOrDimensionTableName(), timePartSpecfromXTimePartSpec(xpart.getTimePartitionSpec()), nonTimePartSpecfromXNonTimePartSpec(xpart.getNonTimePartitionSpec()), UpdatePeriod.valueOf(xpart.getUpdatePeriod().name())); partDesc.setPartParams(mapFromXProperties(xpart.getPartitionParameters())); partDesc.setSerdeParams(mapFromXProperties(xpart.getSerdeParameters())); partDesc.setLocation(xpart.getLocation()); partDesc.setInputFormat(xpart.getInputFormat()); partDesc.setOutputFormat(xpart.getOutputFormat()); partDesc.setSerializationLib(xpart.getSerdeClassname()); return partDesc; }