/** * Get an instance of {@link HiveRegister}. * * @param props A {@link State} object. To get a specific implementation of {@link HiveRegister}, * specify property {@link #HIVE_REGISTER_TYPE} as the class name. Otherwise, {@link #DEFAULT_HIVE_REGISTER_TYPE} * will be returned. This {@link State} object is also used to instantiate the {@link HiveRegister} object. */ public static HiveRegister get(State props) { Optional<String> metastoreUri = Optional.fromNullable(props.getProperties().getProperty(HIVE_METASTORE_URI_KEY)); return get(props, metastoreUri); }
/** * Get an instance of {@link HiveRegister}. * * @param props A {@link State} object. To get a specific implementation of {@link HiveRegister}, * specify property {@link #HIVE_REGISTER_TYPE} as the class name. Otherwise, {@link #DEFAULT_HIVE_REGISTER_TYPE} * will be returned. This {@link State} object is also used to instantiate the {@link HiveRegister} object. */ public static HiveRegister get(State props, Optional<String> metastoreURI) { return get(props.getProp(HIVE_REGISTER_TYPE, DEFAULT_HIVE_REGISTER_TYPE), props, metastoreURI); }
public HiveRegistrationCompactorListener(Properties properties) { State state = new State(properties); this.hiveRegister = HiveRegister.get(state); this.hiveRegistrationPolicy = HiveRegistrationPolicyBase.getPolicy(state); }
/** * @param state This is a Job State */ public HiveRegistrationPublisher(State state) { super(state); this.hiveRegister = this.closer.register(HiveRegister.get(state)); this.hivePolicyExecutor = ExecutorsUtils.loggingDecorator(Executors.newFixedThreadPool(new HiveRegProps(state).getNumThreads(), ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of("HivePolicyExecutor-%d")))); this.metricContext = Instrumented.getMetricContext(state, HiveRegistrationPublisher.class); isPathDedupeEnabled = state.getPropAsBoolean(PATH_DEDUPE_ENABLED, this.DEFAULT_PATH_DEDUPE_ENABLED); }
@Override public void execute() throws IOException { try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropTableIfExists(this.table.getDbName(), this.table.getTableName()); } }
/** * Register the given {@link Path}s. * * @param paths The {@link Path}s to be registered. * @param state A {@link State} which will be used to instantiate a {@link HiveRegister} and a * {@link HiveRegistrationPolicy} for registering the given The {@link Path}s. */ public static void register(Iterable<String> paths, State state) throws IOException { try (HiveRegister hiveRegister = HiveRegister.get(state)) { HiveRegistrationPolicy policy = HiveRegistrationPolicyBase.getPolicy(state); for (String path : paths) { for (HiveSpec spec : policy.getHiveSpecs(new Path(path))) { hiveRegister.register(spec); } } } } }
public void onCompactionJobComplete(FileSystemDataset dataset) throws IOException { if (state.contains(ConfigurationKeys.HIVE_REGISTRATION_POLICY)) { HiveRegister hiveRegister = HiveRegister.get(state); HiveRegistrationPolicy hiveRegistrationPolicy = HiveRegistrationPolicyBase.getPolicy(state); CompactionPathParser.CompactionParserResult result = new CompactionPathParser(state).parse(dataset); List<String> paths = new ArrayList<>(); for (HiveSpec spec : hiveRegistrationPolicy.getHiveSpecs(new Path(result.getDstAbsoluteDir()))) { hiveRegister.register(spec); paths.add(spec.getPath().toUri().toASCIIString()); log.info("Hive registration is done for {}", result.getDstAbsoluteDir()); } // submit events for hive registration if (eventSubmitter != null) { Map<String, String> eventMetadataMap = ImmutableMap.of(CompactionSlaEventHelper.DATASET_URN, dataset.datasetURN(), CompactionSlaEventHelper.HIVE_REGISTRATION_PATHS, Joiner.on(',').join(paths)); this.eventSubmitter.submit(CompactionSlaEventHelper.COMPACTION_HIVE_REGISTRATION_EVENT, eventMetadataMap); } } }
@Override public void execute() throws IOException { if (this.verifyBeforeRegistering) { if (!this.hiveSpec.getTable().getLocation().isPresent()) { throw getException("Table does not have a location parameter."); } Path tablePath = new Path(this.hiveSpec.getTable().getLocation().get()); FileSystem fs = this.hiveSpec.getPath().getFileSystem(new Configuration()); if (!fs.exists(tablePath)) { throw getException(String.format("Table location %s does not exist.", tablePath)); } if (this.hiveSpec.getPartition().isPresent()) { if (!this.hiveSpec.getPartition().get().getLocation().isPresent()) { throw getException("Partition does not have a location parameter."); } Path partitionPath = new Path(this.hiveSpec.getPartition().get().getLocation().get()); if (!fs.exists(this.hiveSpec.getPath())) { throw getException(String.format("Partition location %s does not exist.", partitionPath)); } } } try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { log.info("Registering Hive Spec " + this.hiveSpec); ListenableFuture<Void> future = hiveRegister.register(this.hiveSpec); future.get(); } catch (InterruptedException | ExecutionException ie) { throw new IOException("Hive registration was interrupted.", ie); } }
@Override public void execute() throws IOException { HiveTable hiveTable = HiveMetaStoreUtils.getHiveTable(this.table); try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropPartitionIfExists(this.partition.getDbName(), this.partition.getTableName(), hiveTable.getPartitionKeys(), this.partition.getValues()); } }
/** * Get an instance of {@link HiveRegister}. * * @param props A {@link State} object. To get a specific implementation of {@link HiveRegister}, * specify property {@link #HIVE_REGISTER_TYPE} as the class name. Otherwise, {@link #DEFAULT_HIVE_REGISTER_TYPE} * will be returned. This {@link State} object is also used to instantiate the {@link HiveRegister} object. */ public static HiveRegister get(State props) { Optional<String> metastoreUri = Optional.fromNullable(props.getProperties().getProperty(HIVE_METASTORE_URI_KEY)); return get(props, metastoreUri); }
/** * Get an instance of {@link HiveRegister}. * * @param props A {@link State} object. To get a specific implementation of {@link HiveRegister}, * specify property {@link #HIVE_REGISTER_TYPE} as the class name. Otherwise, {@link #DEFAULT_HIVE_REGISTER_TYPE} * will be returned. This {@link State} object is also used to instantiate the {@link HiveRegister} object. */ public static HiveRegister get(State props, Optional<String> metastoreURI) { return get(props.getProp(HIVE_REGISTER_TYPE, DEFAULT_HIVE_REGISTER_TYPE), props, metastoreURI); }
public HiveRegistrationCompactorListener(Properties properties) { State state = new State(properties); this.hiveRegister = HiveRegister.get(state); this.hiveRegistrationPolicy = HiveRegistrationPolicyBase.getPolicy(state); }
/** * @param state This is a Job State */ public HiveRegistrationPublisher(State state) { super(state); this.hiveRegister = this.closer.register(HiveRegister.get(state)); this.hivePolicyExecutor = ExecutorsUtils.loggingDecorator(Executors.newFixedThreadPool(new HiveRegProps(state).getNumThreads(), ExecutorsUtils.newThreadFactory(Optional.of(log), Optional.of("HivePolicyExecutor-%d")))); this.metricContext = Instrumented.getMetricContext(state, HiveRegistrationPublisher.class); isPathDedupeEnabled = state.getPropAsBoolean(PATH_DEDUPE_ENABLED, this.DEFAULT_PATH_DEDUPE_ENABLED); }
@Override public void execute() throws IOException { try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropTableIfExists(this.table.getDbName(), this.table.getTableName()); } }
/** * Register the given {@link Path}s. * * @param paths The {@link Path}s to be registered. * @param state A {@link State} which will be used to instantiate a {@link HiveRegister} and a * {@link HiveRegistrationPolicy} for registering the given The {@link Path}s. */ public static void register(Iterable<String> paths, State state) throws IOException { try (HiveRegister hiveRegister = HiveRegister.get(state)) { HiveRegistrationPolicy policy = HiveRegistrationPolicyBase.getPolicy(state); for (String path : paths) { for (HiveSpec spec : policy.getHiveSpecs(new Path(path))) { hiveRegister.register(spec); } } } } }
public void onCompactionJobComplete(FileSystemDataset dataset) throws IOException { if (state.contains(ConfigurationKeys.HIVE_REGISTRATION_POLICY)) { HiveRegister hiveRegister = HiveRegister.get(state); HiveRegistrationPolicy hiveRegistrationPolicy = HiveRegistrationPolicyBase.getPolicy(state); CompactionPathParser.CompactionParserResult result = new CompactionPathParser(state).parse(dataset); List<String> paths = new ArrayList<>(); for (HiveSpec spec : hiveRegistrationPolicy.getHiveSpecs(new Path(result.getDstAbsoluteDir()))) { hiveRegister.register(spec); paths.add(spec.getPath().toUri().toASCIIString()); log.info("Hive registration is done for {}", result.getDstAbsoluteDir()); } // submit events for hive registration if (eventSubmitter != null) { Map<String, String> eventMetadataMap = ImmutableMap.of(CompactionSlaEventHelper.DATASET_URN, dataset.datasetURN(), CompactionSlaEventHelper.HIVE_REGISTRATION_PATHS, Joiner.on(',').join(paths)); this.eventSubmitter.submit(CompactionSlaEventHelper.COMPACTION_HIVE_REGISTRATION_EVENT, eventMetadataMap); } } }
@Override public void execute() throws IOException { if (this.verifyBeforeRegistering) { if (!this.hiveSpec.getTable().getLocation().isPresent()) { throw getException("Table does not have a location parameter."); } Path tablePath = new Path(this.hiveSpec.getTable().getLocation().get()); FileSystem fs = this.hiveSpec.getPath().getFileSystem(new Configuration()); if (!fs.exists(tablePath)) { throw getException(String.format("Table location %s does not exist.", tablePath)); } if (this.hiveSpec.getPartition().isPresent()) { if (!this.hiveSpec.getPartition().get().getLocation().isPresent()) { throw getException("Partition does not have a location parameter."); } Path partitionPath = new Path(this.hiveSpec.getPartition().get().getLocation().get()); if (!fs.exists(this.hiveSpec.getPath())) { throw getException(String.format("Partition location %s does not exist.", partitionPath)); } } } try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { log.info("Registering Hive Spec " + this.hiveSpec); ListenableFuture<Void> future = hiveRegister.register(this.hiveSpec); future.get(); } catch (InterruptedException | ExecutionException ie) { throw new IOException("Hive registration was interrupted.", ie); } }
@Override public void execute() throws IOException { HiveTable hiveTable = HiveMetaStoreUtils.getHiveTable(this.table); try (HiveRegister hiveRegister = HiveRegister.get(this.props, this.metastoreURI)) { hiveRegister.dropPartitionIfExists(this.partition.getDbName(), this.partition.getTableName(), hiveTable.getPartitionKeys(), this.partition.getValues()); } }