/** * Register the given {@link Path}s. * * @param paths The {@link Path}s to be registered. * @param state A {@link State} which will be used to instantiate a {@link HiveRegister} and a * {@link HiveRegistrationPolicy} for registering the given The {@link Path}s. */ public static void register(Iterable<String> paths, State state) throws IOException { try (HiveRegister hiveRegister = HiveRegister.get(state)) { HiveRegistrationPolicy policy = HiveRegistrationPolicyBase.getPolicy(state); for (String path : paths) { for (HiveSpec spec : policy.getHiveSpecs(new Path(path))) { hiveRegister.register(spec); } } } } }
/** * Add a partition to a table if not exists, or alter a partition if exists. * * @param table the {@link HiveTable} to which the partition belongs. * @param partition a {@link HivePartition} to which the existing partition should be updated. * @throws IOException */ public void addOrAlterPartition(HiveTable table, HivePartition partition) throws IOException { if (!addPartitionIfNotExists(table, partition)) { alterPartition(table, partition); } }
/** * Create a table if not exists, or alter a table if exists. * * @param table a {@link HiveTable} to be created or altered * @throws IOException */ public void createOrAlterTable(HiveTable table) throws IOException { if (!createTableIfNotExists(table)) { alterTable(table); } }
@Override public void execute() throws IOException { try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropTableIfExists(this.table.getDbName(), this.table.getTableName()); } }
/** * Get an instance of {@link HiveRegister}. * * @param props A {@link State} object. To get a specific implementation of {@link HiveRegister}, * specify property {@link #HIVE_REGISTER_TYPE} as the class name. Otherwise, {@link #DEFAULT_HIVE_REGISTER_TYPE} * will be returned. This {@link State} object is also used to instantiate the {@link HiveRegister} object. */ public static HiveRegister get(State props, Optional<String> metastoreURI) { return get(props.getProp(HIVE_REGISTER_TYPE, DEFAULT_HIVE_REGISTER_TYPE), props, metastoreURI); }
@Override public void execute() throws IOException { HiveTable hiveTable = HiveMetaStoreUtils.getHiveTable(this.table); try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropPartitionIfExists(this.partition.getDbName(), this.partition.getTableName(), hiveTable.getPartitionKeys(), this.partition.getValues()); } }
if (this.hiveRegister.getProps().getUpstreamDataAttrName().isPresent()) { for (String attrName: LIST_SPLITTER_COMMA.splitToList(this.hiveRegister.getProps().getUpstreamDataAttrName().get())){ if (state.contains(attrName)) { taskSpecificState.appendToListProp(HiveMetaStoreUtils.RUNTIME_PROPS, try { for (HiveSpec spec : completionService.take().get()) { this.hiveRegister.register(spec);
@Override public void onDatasetCompactionCompletion(Dataset dataset) throws Exception { for (HiveSpec spec : this.hiveRegistrationPolicy.getHiveSpecs(dataset.outputPath())) { this.hiveRegister.register(spec); } } }
@Override public boolean execute(HiveRegister register) throws IOException { register.dropPartitionIfExists(this.dbName, this.tableName, this.partitionKeys, this.partitionValues); return true; }
@Override public boolean apply(HiveRegister register) { try { return !register.existsPartition(this.dbName, this.tableName, this.partitionKeys, this.partitionValues); } catch (IOException e) { throw Throwables.propagate(e); } } }
@Override public boolean execute(HiveRegister register) throws IOException { register.dropTableIfExists(this.dbName, this.tableName); return true; }
/** * Get an instance of {@link HiveRegister}. * * @param props A {@link State} object. To get a specific implementation of {@link HiveRegister}, * specify property {@link #HIVE_REGISTER_TYPE} as the class name. Otherwise, {@link #DEFAULT_HIVE_REGISTER_TYPE} * will be returned. This {@link State} object is also used to instantiate the {@link HiveRegister} object. */ public static HiveRegister get(State props) { Optional<String> metastoreUri = Optional.fromNullable(props.getProperties().getProperty(HIVE_METASTORE_URI_KEY)); return get(props, metastoreUri); }
@Override public void execute() throws IOException { try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropTableIfExists(this.table.getDbName(), this.table.getTableName()); } }
@Override public void execute() throws IOException { HiveTable hiveTable = HiveMetaStoreUtils.getHiveTable(this.table); try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropPartitionIfExists(this.partition.getDbName(), this.partition.getTableName(), hiveTable.getPartitionKeys(), this.partition.getValues()); } }
if (this.hiveRegister.getProps().getUpstreamDataAttrName().isPresent()) { for (String attrName: LIST_SPLITTER_COMMA.splitToList(this.hiveRegister.getProps().getUpstreamDataAttrName().get())){ if (state.contains(attrName)) { taskSpecificState.appendToListProp(HiveMetaStoreUtils.RUNTIME_PROPS, try { for (HiveSpec spec : completionService.take().get()) { this.hiveRegister.register(spec);
@Override public void onDatasetCompactionCompletion(Dataset dataset) throws Exception { for (HiveSpec spec : this.hiveRegistrationPolicy.getHiveSpecs(dataset.outputPath())) { this.hiveRegister.register(spec); } } }
@Override public boolean execute(HiveRegister register) throws IOException { register.dropPartitionIfExists(this.dbName, this.tableName, this.partitionKeys, this.partitionValues); return true; }
@Override public boolean apply(HiveRegister register) { try { return !register.existsPartition(this.dbName, this.tableName, this.partitionKeys, this.partitionValues); } catch (IOException e) { throw Throwables.propagate(e); } } }
@Override public boolean execute(HiveRegister register) throws IOException { register.dropTableIfExists(this.dbName, this.tableName); return true; }
public void onCompactionJobComplete(FileSystemDataset dataset) throws IOException { if (state.contains(ConfigurationKeys.HIVE_REGISTRATION_POLICY)) { HiveRegister hiveRegister = HiveRegister.get(state); HiveRegistrationPolicy hiveRegistrationPolicy = HiveRegistrationPolicyBase.getPolicy(state); CompactionPathParser.CompactionParserResult result = new CompactionPathParser(state).parse(dataset); List<String> paths = new ArrayList<>(); for (HiveSpec spec : hiveRegistrationPolicy.getHiveSpecs(new Path(result.getDstAbsoluteDir()))) { hiveRegister.register(spec); paths.add(spec.getPath().toUri().toASCIIString()); log.info("Hive registration is done for {}", result.getDstAbsoluteDir()); } // submit events for hive registration if (eventSubmitter != null) { Map<String, String> eventMetadataMap = ImmutableMap.of(CompactionSlaEventHelper.DATASET_URN, dataset.datasetURN(), CompactionSlaEventHelper.HIVE_REGISTRATION_PATHS, Joiner.on(',').join(paths)); this.eventSubmitter.submit(CompactionSlaEventHelper.COMPACTION_HIVE_REGISTRATION_EVENT, eventMetadataMap); } } }