@Override public Map<String, String> getParameters() { return partitionList.get(index).getParameters(); }
public static boolean isArchived(Partition part) { Map<String, String> params = part.getParameters(); return (params != null && "TRUE".equalsIgnoreCase(params.get(hive_metastoreConstants.IS_ARCHIVED))); }
private boolean isPartitionExternal() { Map<String, String> params = partition.getParameters(); return params.containsKey("EXTERNAL") && params.get("EXTERNAL").equalsIgnoreCase("TRUE"); } }
public static Path getOriginalLocation(Partition part) { Map<String, String> params = part.getParameters(); assert(isArchived(part)); String originalLocation = params.get(hive_metastoreConstants.ORIGINAL_LOCATION); assert( originalLocation != null); return new Path(originalLocation); }
private boolean isCurrentStatsValidForTheQuery(Partition part, long partWriteId, String queryValidWriteIdList, boolean isCompleteStatsWriter) throws MetaException { return isCurrentStatsValidForTheQuery(conf, part.getParameters(), partWriteId, queryValidWriteIdList, isCompleteStatsWriter); }
public void exec(JobContext context, Partition partition, Path partPath) throws IOException { // LOG.info("Archiving partition ["+partPath.toString()+"]"); makeHar(context, partPath.toUri().toString(), harFile(partPath)); partition.getParameters().put(hive_metastoreConstants.IS_ARCHIVED, "true"); }
public static int getArchivingLevel(Partition part) throws MetaException { if (!isArchived(part)) { throw new MetaException("Getting level of unarchived partition"); } String lv = part.getParameters().get(ARCHIVING_LEVEL); if (lv != null) { return Integer.parseInt(lv); } // partitions archived before introducing multiple archiving return part.getValues().size(); }
public Properties getSchemaFromTableSchema(Properties tblSchema) { return MetaStoreUtils.getPartSchemaFromTableSchema(tPartition.getSd(), tPartition.getParameters(), tblSchema); }
@Override public synchronized void updatePartitionStatistics(String databaseName, String tableName, String partitionName, Function<PartitionStatistics, PartitionStatistics> update) { PartitionStatistics currentStatistics = requireNonNull( getPartitionStatistics(databaseName, tableName, ImmutableSet.of(partitionName)).get(partitionName), "getPartitionStatistics() returned null"); PartitionStatistics updatedStatistics = update.apply(currentStatistics); List<Partition> partitions = getPartitionsByNames(databaseName, tableName, ImmutableList.of(partitionName)); if (partitions.size() != 1) { throw new PrestoException(HIVE_METASTORE_ERROR, "Metastore returned multiple partitions for name: " + partitionName); } Partition originalPartition = getOnlyElement(partitions); Partition modifiedPartition = originalPartition.deepCopy(); HiveBasicStatistics basicStatistics = updatedStatistics.getBasicStatistics(); modifiedPartition.setParameters(updateStatisticsParameters(modifiedPartition.getParameters(), basicStatistics)); alterPartitionWithoutStatistics(databaseName, tableName, modifiedPartition); Map<String, HiveType> columns = modifiedPartition.getSd().getCols().stream() .collect(toImmutableMap(FieldSchema::getName, schema -> HiveType.valueOf(schema.getType()))); setPartitionColumnStatistics(databaseName, tableName, partitionName, columns, updatedStatistics.getColumnStatistics(), basicStatistics.getRowCount()); Set<String> removedStatistics = difference(currentStatistics.getColumnStatistics().keySet(), updatedStatistics.getColumnStatistics().keySet()); removedStatistics.forEach(column -> deletePartitionColumnStatistics(databaseName, tableName, partitionName, column)); }
public Partish buildPartition() { Partition partition = Mockito.mock(Partition.class); org.apache.hadoop.hive.metastore.api.Partition tpartition = Mockito.mock(org.apache.hadoop.hive.metastore.api.Partition.class); doReturn(tpartition).when(partition).getTPartition(); doReturn(params).when(tpartition).getParameters(); return Partish.buildFor(null, partition); } }
public static org.apache.hadoop.hive.metastore.api.Partition toMetastoreApiPartition(PartitionWithStatistics partitionWithStatistics) { org.apache.hadoop.hive.metastore.api.Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); partition.setParameters(updateStatisticsParameters(partition.getParameters(), partitionWithStatistics.getStatistics().getBasicStatistics())); return partition; }
public static String createPartitionObjJson(Partition partitionObj) throws TException { //Note: The parameters of the Partition object will be removed in the filter if it matches // any pattern provided through EVENT_NOTIFICATION_PARAMETERS_EXCLUDE_PATTERNS filterMapkeys(partitionObj.getParameters(), paramsFilter); TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); return serializer.toString(partitionObj, "UTF-8"); }
public static Partition fromMetastoreApiPartition(org.apache.hadoop.hive.metastore.api.Partition partition) { StorageDescriptor storageDescriptor = partition.getSd(); if (storageDescriptor == null) { throw new PrestoException(HIVE_INVALID_METADATA, "Partition does not contain a storage descriptor: " + partition); } Partition.Builder partitionBuilder = Partition.builder() .setDatabaseName(partition.getDbName()) .setTableName(partition.getTableName()) .setValues(partition.getValues()) .setColumns(storageDescriptor.getCols().stream() .map(ThriftMetastoreUtil::fromMetastoreApiFieldSchema) .collect(toList())) .setParameters(partition.getParameters()); fromMetastoreApiStorageDescriptor(storageDescriptor, partitionBuilder.getStorageBuilder(), format("%s.%s", partition.getTableName(), partition.getValues())); return partitionBuilder.build(); }
@Override public synchronized void alterPartition(String databaseName, String tableName, PartitionWithStatistics partitionWithStatistics) { Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName()); partitions.put(partitionKey, partition); partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics()); }
@Override public synchronized void addPartitions(String databaseName, String tableName, List<PartitionWithStatistics> partitionsWithStatistics) { for (PartitionWithStatistics partitionWithStatistics : partitionsWithStatistics) { Partition partition = toMetastoreApiPartition(partitionWithStatistics.getPartition()); if (partition.getParameters() == null) { partition.setParameters(ImmutableMap.of()); } PartitionName partitionKey = PartitionName.partition(databaseName, tableName, partitionWithStatistics.getPartitionName()); partitions.put(partitionKey, partition); partitionColumnStatistics.put(partitionKey, partitionWithStatistics.getStatistics()); } }
public static Properties getPartitionMetadata( org.apache.hadoop.hive.metastore.api.Partition partition, org.apache.hadoop.hive.metastore.api.Table table) { return MetaStoreUtils .getSchema(partition.getSd(), partition.getSd(), partition .getParameters(), table.getDbName(), table.getTableName(), table.getPartitionKeys()); }
private static void getPartitionMetaDataInformation(StringBuilder tableInfo, Partition part) { formatOutput("Partition Value:", part.getValues().toString(), tableInfo); formatOutput("Database:", part.getTPartition().getDbName(), tableInfo); formatOutput("Table:", part.getTable().getTableName(), tableInfo); formatOutput("CreateTime:", formatDate(part.getTPartition().getCreateTime()), tableInfo); formatOutput("LastAccessTime:", formatDate(part.getTPartition().getLastAccessTime()), tableInfo); formatOutput("Location:", part.getLocation(), tableInfo); if (part.getTPartition().getParameters().size() > 0) { tableInfo.append("Partition Parameters:").append(LINE_DELIM); displayAllParameters(part.getTPartition().getParameters(), tableInfo); } }
private void verifyStatsUpToDate(String tbl, String part, ArrayList<String> cols, IMetaStoreClient msClient, boolean isUpToDate) throws Exception { Partition partition = msClient.getPartition(ss.getCurrentDatabase(), tbl, part); verifyStatsUpToDate(partition.getParameters(), cols, isUpToDate); }
private void setPartitionSkipProperty( IMetaStoreClient msClient, String tblName, String partName, String val) throws Exception { Partition part = msClient.getPartition(ss.getCurrentDatabase(), tblName, partName); part.getParameters().put(StatsUpdaterThread.SKIP_STATS_AUTOUPDATE_PROPERTY, val); msClient.alter_partition(part.getCatName(), part.getDbName(), tblName, part); }
private static void makeTestChangesOnPartition(Partition partition) { partition.getParameters().put("hmsTestParam001", "testValue001"); partition.setCreateTime(NEW_CREATE_TIME); partition.setLastAccessTime(NEW_CREATE_TIME); partition.getSd().setLocation(partition.getSd().getLocation()+"/hh=01"); partition.getSd().getCols().add(new FieldSchema("newcol", "string", "")); }