@Override public long getUpdateTime(Partition partition) throws UpdateNotFoundException { // TODO if a table/partition is registered by gobblin an update time will be made available in table properties // Use the update time instead of create time return TimeUnit.MILLISECONDS.convert(partition.getTPartition().getCreateTime(), TimeUnit.SECONDS); }
private static State getPartitionProps(Partition partition) { State partitionProps = new State(); for (Map.Entry<String, String> entry : partition.getParameters().entrySet()) { partitionProps.setProp(entry.getKey(), entry.getValue()); } if (partition.isSetCreateTime()) { partitionProps.setProp(HiveConstants.CREATE_TIME, partition.getCreateTime()); } if (partition.isSetLastAccessTime()) { partitionProps.setProp(HiveConstants.LAST_ACCESS_TIME, partition.getCreateTime()); } return partitionProps; }
@VisibleForTesting public static long getCreateTime(Partition partition) { // If create time is set, use it. // .. this is always set if HiveJDBC or Hive mestastore is used to create partition. // .. it might not be set (ie. equals 0) if Thrift API call is used to create partition. if (partition.getTPartition().getCreateTime() > 0) { return TimeUnit.MILLISECONDS.convert(partition.getTPartition().getCreateTime(), TimeUnit.SECONDS); } // Try to use distcp-ng registration generation time if it is available else if (partition.getTPartition().isSetParameters() && partition.getTPartition().getParameters().containsKey(DISTCP_REGISTRATION_GENERATION_TIME_KEY)) { log.debug("Did not find createTime in Hive partition, used distcp registration generation time."); return Long.parseLong(partition.getTPartition().getParameters().get(DISTCP_REGISTRATION_GENERATION_TIME_KEY)); } else { log.warn(String.format("Could not find create time for partition %s. Will return createTime as 0", partition.getCompleteName())); return 0; } }
/** * Sets create time if not already set. */ private Partition getPartitionWithCreateTime(Partition partition, int createTime) { if (partition.isSetCreateTime() && partition.getCreateTime() > 0) { return partition; } Partition actualPartition = partition.deepCopy(); actualPartition.setCreateTime(createTime); return actualPartition; }
private static void adjust(HiveMetaStoreClient client, Partition part, String dbName, String tblName, boolean isThriftClient) throws TException { Partition part_get = client.getPartition(dbName, tblName, part.getValues()); if (isThriftClient) { part.setCreateTime(part_get.getCreateTime()); part.putToParameters(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.DDL_TIME, Long.toString(part_get.getCreateTime())); } part.setWriteId(part_get.getWriteId()); }
private void assertPartitionUnchanged(Partition partition, List<String> testValues, List<String> partCols) throws MetaException { assertFalse(partition.getParameters().containsKey("hmsTestParam001")); List<String> expectedKVPairs = new ArrayList<>(); for (int i = 0; i < partCols.size(); ++i) { expectedKVPairs.add(partCols.get(i) + "=" + testValues.get(i)); } String partPath = expectedKVPairs.stream().collect(joining("/")); assertTrue(partition.getSd().getLocation().equals(metaStore.getWarehouseRoot() + "/testpartdb.db/testparttable/" + partPath)); assertNotEquals(NEW_CREATE_TIME, partition.getCreateTime()); assertNotEquals(NEW_CREATE_TIME, partition.getLastAccessTime()); assertEquals(2, partition.getSd().getCols().size()); }
private void assertPartitionChanged(Partition partition, List<String> testValues, List<String> partCols) throws MetaException { assertEquals("testValue001", partition.getParameters().get("hmsTestParam001")); List<String> expectedKVPairs = new ArrayList<>(); for (int i = 0; i < partCols.size(); ++i) { expectedKVPairs.add(partCols.get(i) + "=" + testValues.get(i)); } String partPath = expectedKVPairs.stream().collect(joining("/")); assertTrue(partition.getSd().getLocation().equals(metaStore.getWarehouseRoot() + "/testpartdb.db/testparttable/" + partPath + "/hh=01")); assertEquals(NEW_CREATE_TIME, partition.getCreateTime()); assertEquals(NEW_CREATE_TIME, partition.getLastAccessTime()); assertEquals(3, partition.getSd().getCols().size()); }
private static void getPartitionMetaDataInformation(StringBuilder tableInfo, Partition part) { formatOutput("Partition Value:", part.getValues().toString(), tableInfo); formatOutput("Database:", part.getTPartition().getDbName(), tableInfo); formatOutput("Table:", part.getTable().getTableName(), tableInfo); formatOutput("CreateTime:", formatDate(part.getTPartition().getCreateTime()), tableInfo); formatOutput("LastAccessTime:", formatDate(part.getTPartition().getLastAccessTime()), tableInfo); formatOutput("Location:", part.getLocation(), tableInfo); if (part.getTPartition().getParameters().size() > 0) { tableInfo.append("Partition Parameters:").append(LINE_DELIM); displayAllParameters(part.getTPartition().getParameters(), tableInfo); } }
private static void getPartitionMetaDataInformation(StringBuilder tableInfo, Partition part) { formatOutput("Partition Value:", part.getValues().toString(), tableInfo); formatOutput("Database:", part.getTPartition().getDbName(), tableInfo); formatOutput("Table:", part.getTable().getTableName(), tableInfo); formatOutput("CreateTime:", formatDate(part.getTPartition().getCreateTime()), tableInfo); formatOutput("LastAccessTime:", formatDate(part.getTPartition().getLastAccessTime()), tableInfo); formatOutput("Location:", part.getLocation(), tableInfo); if (part.getTPartition().getParameters().size() > 0) { tableInfo.append("Partition Parameters:").append(LINE_DELIM); displayAllParameters(part.getTPartition().getParameters(), tableInfo); } }
HCatPartition(HCatTable hcatTable, Partition partition) throws HCatException { this.hcatTable = hcatTable; this.tableName = partition.getTableName(); this.dbName = partition.getDbName(); this.createTime = partition.getCreateTime(); this.lastAccessTime = partition.getLastAccessTime(); this.parameters = partition.getParameters(); this.values = partition.getValues(); if (hcatTable != null && partition.getValuesSize() != hcatTable.getPartCols().size()) { throw new HCatException("Mismatched number of partition columns between table:" + hcatTable.getDbName() + "." + hcatTable.getTableName() + " and partition " + partition.getValues()); } this.sd = partition.getSd(); this.columns = getColumns(this.sd); }
/** * Testing * renamePartition(String,String,List(String),Partition) -> * renamePartition(String,String,List(String),Partition). */ @Test public void testRenamePartition() throws Exception { List<List<String>> oldValues = createTable4PartColsParts(client); List<List<String>> newValues = new ArrayList<>(); List<String> newVal = Lists.newArrayList("2018", "01", "16"); newValues.addAll(oldValues.subList(0, 3)); newValues.add(newVal); List<Partition> oldParts = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1); Partition partToRename = oldParts.get(3); partToRename.setValues(newVal); makeTestChangesOnPartition(partToRename); client.renamePartition(DB_NAME, TABLE_NAME, oldValues.get(3), partToRename); List<Partition> newParts = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1); assertPartitionsHaveCorrectValues(newParts, newValues); //Asserting other partition parameters can also be changed, but not the location assertFalse(newParts.get(3).getSd().getLocation().endsWith("hh=01")); assertEquals(3, newParts.get(3).getSd().getCols().size()); assertEquals("testValue001", newParts.get(3).getParameters().get("hmsTestParam001")); assertEquals(NEW_CREATE_TIME, newParts.get(3).getCreateTime()); assertEquals(NEW_CREATE_TIME, newParts.get(3).getLastAccessTime()); assertTrue(client.listPartitions(DB_NAME, TABLE_NAME, oldValues.get(3), (short)-1).isEmpty()); }
public Object getFieldValue(_Fields field) { switch (field) { case VALUES: return getValues(); case DB_NAME: return getDbName(); case TABLE_NAME: return getTableName(); case CREATE_TIME: return getCreateTime(); case LAST_ACCESS_TIME: return getLastAccessTime(); case SD: return getSd(); case PARAMETERS: return getParameters(); case PRIVILEGES: return getPrivileges(); case CAT_NAME: return getCatName(); case WRITE_ID: return getWriteId(); case IS_STATS_COMPLIANT: return isIsStatsCompliant(); case COL_STATS: return getColStats(); } throw new IllegalStateException(); }
hiveDatasetVersion.getPartition().getTable().getDbName(), hiveDatasetVersion.getPartition().getTable().getTableName(), sourcePartition.getCreateTime(), sourcePartition.getLastAccessTime(), sourcePartition.getSd(),
/** * Convert a {@link Partition} into a {@link HivePartition}. */ public static HivePartition getHivePartition(Partition partition) { State partitionProps = getPartitionProps(partition); State storageProps = getStorageProps(partition.getSd()); State serDeProps = getSerDeProps(partition.getSd().getSerdeInfo()); HivePartition hivePartition = new HivePartition.Builder().withDbName(partition.getDbName()).withTableName(partition.getTableName()) .withPartitionValues(partition.getValues()).withProps(partitionProps).withStorageProps(storageProps) .withSerdeProps(serDeProps).build(); if (partition.getCreateTime() > 0) { hivePartition.setCreateTime(partition.getCreateTime()); } if (partition.getSd().getCols() != null) { hivePartition.setColumns(getColumns(partition.getSd().getCols())); } if (partition.getSd().getBucketCols() != null) { hivePartition.setBucketColumns(partition.getSd().getBucketCols()); } return hivePartition; }
part.getDbName()); Assert.assertEquals("The last access time is not correct.", 123456, part.getLastAccessTime()); Assert.assertNotEquals(123456, part.getCreateTime()); Assert.assertEquals( "The partition's parameter map should contain the partparamkey - partparamvalue pair.",
Partition origPartition = origPartitions.get(i); PartitionWithoutSD retPartition = partitionWithoutSDS.get(i); Assert.assertEquals(origPartition.getCreateTime(), retPartition.getCreateTime()); Assert.assertEquals(origPartition.getLastAccessTime(), retPartition.getLastAccessTime()); Assert.assertEquals(origPartition.getSd().getLocation(),
Assert.assertEquals("The last access time is not correct.", DEFAULT_CREATE_TIME, part.getLastAccessTime()); Assert.assertNotEquals(DEFAULT_CREATE_TIME, part.getCreateTime()); Assert.assertEquals( "The partition's parameter map should contain the partparamkey - partparamvalue pair.",
private void verifyPartition(Partition partition, Table table, List<String> expectedPartValues, String partitionName) throws Exception { Assert.assertEquals(table.getTableName(), partition.getTableName()); Assert.assertEquals(table.getDbName(), partition.getDbName()); Assert.assertEquals(expectedPartValues, partition.getValues()); Assert.assertNotEquals(0, partition.getCreateTime()); Assert.assertEquals(0, partition.getLastAccessTime()); Assert.assertEquals(1, partition.getParameters().size()); Assert.assertTrue(partition.getParameters().containsKey("transient_lastDdlTime")); StorageDescriptor partitionSD = partition.getSd(); Assert.assertEquals(table.getSd().getLocation() + "/" + partitionName, partitionSD.getLocation()); partition.getSd().setLocation(table.getSd().getLocation()); Assert.assertEquals(table.getSd(), partitionSD); Assert.assertTrue(metaStore.isPathExists(new Path(partitionSD.getLocation()))); }
partition.getLastAccessTime()); Assert.assertNotEquals("The partition's create time should be set.", 0, partition.getCreateTime()); Assert.assertEquals( "The partition has to have the 'transient_lastDdlTime' parameter per default.", 1,
.getCreateTime(), part.getLastAccessTime(), msd, part.getParameters()); return mpart;