private String getDynPartDirectory(List<String> row, List<String> dpColNames) { return FileUtils.makePartName(dpColNames, row); }
public static String makePartName(List<String> partCols, List<String> vals) { return makePartName(partCols, vals, null); }
private String getDynPartDirectory(List<String> row, List<String> dpColNames) { return FileUtils.makePartName(dpColNames, row); }
public static String makePartName(List<Column> partitionColumns, List<String> values) { checkArgument(partitionColumns.size() == values.size()); List<String> partitionColumnNames = partitionColumns.stream().map(Column::getName).collect(toList()); return FileUtils.makePartName(partitionColumnNames, values); }
private String getPartitionName(String databaseName, String tableName, List<String> partitionValues) { Table table = getTable(databaseName, tableName) .orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName))); List<String> columnNames = table.getPartitionColumns().stream() .map(Column::getName) .collect(toImmutableList()); return makePartName(columnNames, partitionValues); }
partition -> makePartName(partitionColumns, partition.getValues()), partition -> getHiveBasicStatistics(partition.getParameters()))); Map<String, OptionalLong> partitionRowCounts = partitionBasicStatistics.entrySet().stream()
List<String> values = partitionAction.getData().getPartition().getValues(); if (!parts.isPresent() || partitionValuesMatch(values, parts.get())) { resultBuilder.add(makePartName(columnNames, values));
throw new PrestoException(ALREADY_EXISTS, "Partition already exists"); String partitionName = FileUtils.makePartName(actualPartitionColumnNames, partitionStringValues);
partitionName = Optional.of(FileUtils.makePartName(partitionColumnNames, partitionValues));
valList.add(partVal); String partName = FileUtils.makePartName(keyList, valList); rqst.setPartname(partName);
values.add(value); outputLocation = FileUtils.makePartName(cols, values);
LOG.debug("DDLTask: Rename Partition is skipped as table {} / partition {} is newer than update", tableName, FileUtils.makePartName(new ArrayList<>(oldPartSpec.keySet()), new ArrayList<>(oldPartSpec.values()))); Partition oldPart = db.getPartition(tbl, oldPartSpec, false); if (oldPart == null) { String partName = FileUtils.makePartName(new ArrayList<String>(oldPartSpec.keySet()), new ArrayList<String>(oldPartSpec.values())); throw new HiveException("Rename partition: source partition [" + partName
partitionSpecs = FileUtils.makePartName(conf.getPartColumns(), values); if (LOG.isInfoEnabled()) { LOG.info("Stats Gathering found a new partition spec = " + partitionSpecs);
valList.add(partVal); String partName = FileUtils.makePartName(keyList, valList); rqst.setPartname(partName);
protected void createDummyPartitionedTable(SchemaTableName tableName, List<ColumnMetadata> columns) throws Exception { doCreateEmptyTable(tableName, ORC, columns); ExtendedHiveMetastore metastoreClient = getMetastoreClient(); Table table = metastoreClient.getTable(tableName.getSchemaName(), tableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(tableName)); List<String> firstPartitionValues = ImmutableList.of("2016-01-01"); List<String> secondPartitionValues = ImmutableList.of("2016-01-02"); String firstPartitionName = makePartName(ImmutableList.of("ds"), firstPartitionValues); String secondPartitionName = makePartName(ImmutableList.of("ds"), secondPartitionValues); List<PartitionWithStatistics> partitions = ImmutableList.of(firstPartitionName, secondPartitionName) .stream() .map(partitionName -> new PartitionWithStatistics(createDummyPartition(table, partitionName), partitionName, PartitionStatistics.empty())) .collect(toImmutableList()); metastoreClient.addPartitions(tableName.getSchemaName(), tableName.getTableName(), partitions); metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), firstPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS); metastoreClient.updatePartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), secondPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS); }
/** * Rename a partition in a table * * @param db * Database to rename the partition. * @param renamePartitionDesc * rename old Partition to new one. * @return Returns 0 when execution succeeds and above 0 if it fails. * @throws HiveException */ private int renamePartition(Hive db, RenamePartitionDesc renamePartitionDesc) throws HiveException { Table tbl = db.getTable(renamePartitionDesc.getTableName()); LinkedHashMap<String, String> oldPartSpec = renamePartitionDesc.getOldPartSpec(); Partition oldPart = db.getPartition(tbl, oldPartSpec, false); if (oldPart == null) { String partName = FileUtils.makePartName(new ArrayList<String>(oldPartSpec.keySet()), new ArrayList<String>(oldPartSpec.values())); throw new HiveException("Rename partition: source partition [" + partName + "] does not exist."); } Partition part = db.getPartition(tbl, oldPartSpec, false); part.setValues(renamePartitionDesc.getNewPartSpec()); db.renamePartition(tbl, oldPartSpec, part); Partition newPart = db .getPartition(tbl, renamePartitionDesc.getNewPartSpec(), false); work.getInputs().add(new ReadEntity(oldPart)); // We've already obtained a lock on the table, don't lock the partition too addIfAbsentByName(new WriteEntity(newPart, WriteEntity.WriteType.DDL_NO_LOCK)); return 0; }
private List<ShowLocksResponseElement> getLocksWithFilterOptions(HiveTxnManager txnMgr, String dbName, String tblName, Map<String, String> partSpec) throws Exception { if (dbName == null && tblName != null) { dbName = SessionState.get().getCurrentDatabase(); } ShowLocksRequest rqst = new ShowLocksRequest(); rqst.setDbname(dbName); rqst.setTablename(tblName); if (partSpec != null) { List<String> keyList = new ArrayList<String>(); List<String> valList = new ArrayList<String>(); for (String partKey : partSpec.keySet()) { String partVal = partSpec.remove(partKey); keyList.add(partKey); valList.add(partVal); } String partName = FileUtils.makePartName(keyList, valList); rqst.setPartname(partName); } ShowLocksResponse rsp = ((DbLockManager)txnMgr.getLockManager()).getLocks(rqst); return rsp.getLocks(); }
private void eraseStatistics(SchemaTableName schemaTableName) { ExtendedHiveMetastore metastoreClient = getMetastoreClient(); metastoreClient.updateTableStatistics(schemaTableName.getSchemaName(), schemaTableName.getTableName(), statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of())); Table table = metastoreClient.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName()) .orElseThrow(() -> new TableNotFoundException(schemaTableName)); List<String> partitionColumns = table.getPartitionColumns().stream() .map(Column::getName) .collect(toImmutableList()); if (!table.getPartitionColumns().isEmpty()) { List<String> partitionNames = metastoreClient.getPartitionNames(schemaTableName.getSchemaName(), schemaTableName.getTableName()) .orElse(ImmutableList.of()); List<Partition> partitions = metastoreClient .getPartitionsByNames(schemaTableName.getSchemaName(), schemaTableName.getTableName(), partitionNames) .entrySet() .stream() .map(Map.Entry::getValue) .filter(Optional::isPresent) .map(Optional::get) .collect(toImmutableList()); for (Partition partition : partitions) { metastoreClient.updatePartitionStatistics( schemaTableName.getSchemaName(), schemaTableName.getTableName(), makePartName(partitionColumns, partition.getValues()), statistics -> new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of())); } } }
LOG.debug("DDLTask: Truncate Table/Partition is skipped as table {} / partition {} is newer than update", tableName, (partSpec == null) ? "null" : FileUtils.makePartName(new ArrayList<>(partSpec.keySet()), new ArrayList<>(partSpec.values())));
String partitionName = makePartName(ImmutableList.of("ds"), partitionValues);