@Override public void setParameters(Map<String, String> parameters) { partition.setParameters(parameters); } @Override public void putToParameters(String key, String value) { partition.putToParameters(key, value);}
@Override public void setParameters(Map<String, String> parameters) { partitionList.get(index).setParameters(parameters); }
static void setPartitionParameters(String PARTITION_PARAMS, boolean convertMapNullsToEmptyStrings, PersistenceManager pm, String partIds, TreeMap<Long, Partition> partitions) throws MetaException { String queryText; queryText = "select \"PART_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from " + PARTITION_PARAMS + "" + " where \"PART_ID\" in (" + partIds + ") and \"PARAM_KEY\" is not null" + " order by \"PART_ID\" asc"; loopJoinOrderedResult(pm, partitions, queryText, 0, new ApplyFunc<Partition>() { @Override public void apply(Partition t, Object[] fields) { t.putToParameters((String)fields[1], extractSqlClob(fields[2])); }}); // Perform conversion of null map values for (Partition t : partitions.values()) { t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } }
@Override public synchronized void updatePartitionStatistics(String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) { PartitionStatistics currentStatistics = requireNonNull( getPartitionStatistics(databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName), "getPartitionStatistics() returned null"); PartitionStatistics updatedStatistics = update.apply(currentStatistics); List<Partition> partitions = getPartitionsByNames(databaseName, tableName, ImmutableList.of(partitionName)); if (partitions.size() != 1) { throw new PrestoException(HIVE_METASTORE_ERROR, "Metastore returned multiple partitions for name: " + partitionName); } Partition originalPartition = getOnlyElement(partitions); Partition modifiedPartition = originalPartition.deepCopy(); HiveBasicStatistics basicStatistics = updatedStatistics.getBasicStatistics(); modifiedPartition.setParameters(updateStatisticsParameters(modifiedPartition.getParameters(), basicStatistics)); alterPartitionWithoutStatistics(databaseName, tableName, modifiedPartition); Map<String, HiveType> columns = modifiedPartition.getSd().getCols().stream() .collect(toImmutableMap(FieldSchema::getName, schema -> HiveType.valueOf(schema.getType()))); setPartitionColumnStatistics(databaseName, tableName, partitionName, columns, updatedStatistics.getColumnStatistics(), basicStatistics.getRowCount()); Set<String> removedStatistics = difference(currentStatistics.getColumnStatistics().keySet(), updatedStatistics.getColumnStatistics().keySet()); removedStatistics.forEach(column -> deletePartitionColumnStatistics(databaseName, tableName, partitionName, column)); }
public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(PartitionWithStatistics partitionWithStatistics) { org.apache.hadoop.hive.metastore.api.Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); partition.setParameters(updateStatisticsParameters(partition.getParameters(), partitionWithStatistics.getStatistics().getBasicStatistics())); return partition; }
@Test public void isFastStatsSameMatching() { Partition oldPartition = new Partition(); Partition newPartition = new Partition(); Map<String, String> stats = new HashMap<>(); Map<String, String> oldParams = new HashMap<>(); Map<String, String> newParams = new HashMap<>(); long testVal = 1; for (String key : FAST_STATS) { oldParams.put(key, String.valueOf(testVal)); newParams.put(key, String.valueOf(testVal)); } oldPartition.setParameters(oldParams); newPartition.setParameters(newParams); assertTrue(MetaStoreServerUtils.isFastStatsSame(oldPartition, newPartition)); }
@Test public void isFastStatsSameNullStatsInNew() { Partition oldPartition = new Partition(); Partition newPartition = new Partition(); Map<String, String> oldParams = new HashMap<>(); Map<String, String> newParams = new HashMap<>(); long testVal = 1; for (String key : FAST_STATS) { oldParams.put(key, String.valueOf(testVal)); newParams.put(key, null); } oldPartition.setParameters(oldParams); newPartition.setParameters(newParams); assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, newPartition)); }
@Test public void isFastStatsSameDifferent() { Partition oldPartition = new Partition(); Partition newPartition = new Partition(); Map<String, String> stats = new HashMap<>(); Map<String, String> oldParams = new HashMap<>(); Map<String, String> newParams = new HashMap<>(); long testVal = 1; for (String key : FAST_STATS) { oldParams.put(key, String.valueOf(testVal)); newParams.put(key, String.valueOf(++testVal)); } oldPartition.setParameters(oldParams); newPartition.setParameters(newParams); assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, newPartition)); }
@Override public synchronized void alterPartition(String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) { Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName()); partitions.put(partitionKey, partition); partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics()); }
@Override public synchronized void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitionsWithStatistics) { for (PartitionWithStatistics partitionWithStatistics : partitionsWithStatistics) { Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName()); partitions.put(partitionKey, partition); partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics()); } }
@Test public void isFastStatsSameWithNoMatchingStats() { Partition oldPartition = new Partition(); Map<String, String> stats = new HashMap<>(); oldPartition.setParameters(stats); assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, null)); stats.put("someKeyThatIsNotInFastStats","value"); oldPartition.setParameters(stats); assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, null)); }
@Test public void testIsOlderThanLookbackForDistcpGenerationTime() throws Exception { long currentTime = System.currentTimeMillis(); // Default lookback time is 3 days long partitionCreateTime = new DateTime(currentTime).minusDays(2).getMillis(); Map<String, String> parameters = Maps.newHashMap(); parameters.put(HiveSource.DISTCP_REGISTRATION_GENERATION_TIME_KEY, partitionCreateTime + ""); org.apache.hadoop.hive.ql.metadata.Partition partition = this.hiveMetastoreTestUtils.createDummyPartition(0); partition.getTPartition().setParameters(parameters); SourceState testState = getTestState("testDb6"); HiveSource source = new HiveSource(); source.initialize(testState); boolean isOlderThanLookback = source.isOlderThanLookback(partition); Assert.assertEquals(isOlderThanLookback, false, "Should create workunits newer than lookback"); }
@Test public void isFastStatsSameMatchingButOnlyOneStat() { Partition oldPartition = new Partition(); Partition newPartition = new Partition(); Map<String, String> randomParams = new HashMap<String, String>(); randomParams.put("randomParam1", "randomVal1"); newPartition.setParameters(randomParams); assertFalse(MetaStoreServerUtils.isFastStatsSame(oldPartition, newPartition)); }
Partition toHivePartition() throws HCatException { Partition hivePtn = new Partition(); hivePtn.setDbName(dbName); hivePtn.setTableName(tableName); hivePtn.setValues(values); hivePtn.setParameters(parameters); if (sd.getLocation() == null) { LOG.warn("Partition location is not set! Attempting to construct default partition location."); try { String partName = Warehouse.makePartName(HCatSchemaUtils.getFieldSchemas(hcatTable.getPartCols()), values); sd.setLocation(new Path(hcatTable.getSd().getLocation(), partName).toString()); } catch(MetaException exception) { throw new HCatException("Could not construct default partition-path for " + hcatTable.getDbName() + "." + hcatTable.getTableName() + "[" + values + "]"); } } hivePtn.setSd(sd); hivePtn.setCreateTime((int) (System.currentTimeMillis() / 1000)); hivePtn.setLastAccessTimeIsSet(false); return hivePtn; }
public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(Partition partition) { org.apache.hadoop.hive.metastore.api.Partition result = new org.apache.hadoop.hive.metastore.api.Partition(); result.setDbName(partition.getDatabaseName()); result.setTableName(partition.getTableName()); result.setValues(partition.getValues()); result.setSd(makeStorageDescriptor(partition.getTableName(), partition.getColumns(), partition.getStorage())); result.setParameters(partition.getParameters()); return result; }
protected Partition newPartition(Table t, String value, List<Order> sortCols) throws Exception { Partition part = new Partition(); part.addToValues(value); part.setDbName(t.getDbName()); part.setTableName(t.getTableName()); part.setSd(newStorageDescriptor(getLocation(t.getTableName(), value), sortCols)); part.setParameters(new HashMap<String, String>()); ms.add_partition(part); return part; }
@Override public Map<String, String> updatePartitionColumnStatistics(ColumnStatistics colStats, List<String> partVals, String validWriteIds, long writeId) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { Map<String, String> newParams = rawStore.updatePartitionColumnStatistics( colStats, partVals, validWriteIds, writeId); // in case of event based cache update, cache is updated during commit txn if (newParams != null && !canUseEvents) { String catName = colStats.getStatsDesc().isSetCatName() ? normalizeIdentifier(colStats.getStatsDesc().getCatName()) : DEFAULT_CATALOG_NAME; String dbName = normalizeIdentifier(colStats.getStatsDesc().getDbName()); String tblName = normalizeIdentifier(colStats.getStatsDesc().getTableName()); if (!shouldCacheTable(catName, dbName, tblName)) { return newParams; } Partition part = getPartition(catName, dbName, tblName, partVals); part.setParameters(newParams); sharedCache.alterPartitionInCache(catName, dbName, tblName, partVals, part); sharedCache.updatePartitionColStatsInCache(catName, dbName, tblName, partVals, colStats.getStatsObj()); } return newParams; }
private static Partition makePartitionObject(String dbName, String tblName, List<String> ptnVals, Table tbl, String ptnLocationSuffix) throws MetaException { Partition part4 = new Partition(); part4.setDbName(dbName); part4.setTableName(tblName); part4.setValues(ptnVals); part4.setParameters(new HashMap<>()); part4.setSd(tbl.getSd().deepCopy()); part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy()); part4.getSd().setLocation(tbl.getSd().getLocation() + ptnLocationSuffix); MetaStoreServerUtils.updatePartitionStatsFast(part4, tbl, warehouse, false, false, null, true); return part4; }
private void add_partition(HiveMetaStoreClient client, Table table, List<String> vals, String location) throws TException { Partition part = new Partition(); part.setDbName(table.getDbName()); part.setTableName(table.getTableName()); part.setValues(vals); part.setParameters(new HashMap<>()); part.setSd(table.getSd().deepCopy()); part.getSd().setSerdeInfo(table.getSd().getSerdeInfo()); part.getSd().setLocation(table.getSd().getLocation() + location); client.add_partition(part); }
private void addPartition(HiveMetaStoreClient client, Table table, List<String> vals, String location) throws TException { Partition part = new Partition(); part.setDbName(table.getDbName()); part.setTableName(table.getTableName()); part.setValues(vals); part.setParameters(new HashMap<String, String>()); part.setSd(table.getSd().deepCopy()); part.getSd().setSerdeInfo(table.getSd().getSerdeInfo()); part.getSd().setLocation(table.getSd().getLocation() + location); client.add_partition(part); } }