/** * @return the {@link Split} object contained in the {@link WorkUnit}. */ public static Optional<Split> getSplit(State workUnit) { return workUnit.contains(SPLIT_KEY) ? Optional.of(GSON.fromJson(workUnit.getProp(SPLIT_KEY), Split.class)) : Optional.<Split>absent(); }
private String getDstBaseDir(State state) { Preconditions.checkArgument(state.contains(MRCompactor.COMPACTION_DEST_DIR), "Missing required property " + MRCompactor.COMPACTION_DEST_DIR); return state.getProp(MRCompactor.COMPACTION_DEST_DIR); }
private String getDstSubDir(State state) { Preconditions.checkArgument(state.contains(MRCompactor.COMPACTION_DEST_SUBDIR), "Missing required property " + MRCompactor.COMPACTION_DEST_SUBDIR); return state.getProp(MRCompactor.COMPACTION_DEST_SUBDIR); } }
protected HiveSnapshotRegistrationPolicy(State props) throws IOException { super(props); this.snapshotPathPattern = props.contains(SNAPSHOT_PATH_PATTERN) ? Optional.of(Pattern.compile(props.getProp(SNAPSHOT_PATH_PATTERN))) : Optional.<Pattern> absent(); }
/** * Check if the given state has lineage info */ public static boolean hasLineageInfo(State state) { return state.contains(getKey(NAME_KEY)); }
private HiveConnection getHiveConnection(Optional<String> proxyUser) throws ClassNotFoundException, SQLException { Class.forName("org.apache.hive.jdbc.HiveDriver"); Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.HIVE_JDBC_URL), "Missing required property " + ComplianceConfigurationKeys.HIVE_JDBC_URL); String url = this.state.getProp(ComplianceConfigurationKeys.HIVE_JDBC_URL); if (proxyUser.isPresent()) { url = url + ComplianceConfigurationKeys.HIVE_SERVER2_PROXY_USER + proxyUser.get(); } return (HiveConnection) DriverManager.getConnection(url); }
public HivePartitionVersionRetentionReaperPolicy(State state, HivePartitionDataset dataset) { super(state, dataset); Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.REAPER_RETENTION_DAYS), "Missing required property " + ComplianceConfigurationKeys.REAPER_RETENTION_DAYS); this.retentionDays = state.getPropAsInt(ComplianceConfigurationKeys.REAPER_RETENTION_DAYS); }
private String getSrcBaseDir(State state) { Preconditions.checkArgument(state.contains(MRCompactor.COMPACTION_INPUT_DIR), "Missing required property " + MRCompactor.COMPACTION_INPUT_DIR); return state.getProp(MRCompactor.COMPACTION_INPUT_DIR); }
/** * Get topic name from a {@link State} object. The {@link State} should contain property * {@link KafkaSource#TOPIC_NAME}. */ public static String getTopicName(State state) { Preconditions.checkArgument(state.contains(KafkaSource.TOPIC_NAME), "Missing configuration property " + KafkaSource.TOPIC_NAME); return state.getProp(KafkaSource.TOPIC_NAME); }
private String getInputDir() { Preconditions.checkArgument(this.state.contains(MRCompactor.COMPACTION_INPUT_DIR), "Missing required property " + MRCompactor.COMPACTION_INPUT_DIR); return this.state.getProp(MRCompactor.COMPACTION_INPUT_DIR); }
@VisibleForTesting GoogleDriveFsHelper(State state, Drive client, Closer closer) { this.closer = closer; this.fileSystem = this.closer.register(new GoogleDriveFileSystem(client)); if (state.contains(BUFFER_SIZE_BYTE)) { this.bufferSizeByte = Optional.of(state.getPropAsInt(BUFFER_SIZE_BYTE)); } else { this.bufferSizeByte = Optional.absent(); } }
private String getBackUpTableLocation(HivePartitionVersion version) { Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.TRASH_DIR), "Missing required property " + ComplianceConfigurationKeys.TRASH_DIR); return StringUtils .join(Arrays.asList(this.state.getProp(ComplianceConfigurationKeys.TRASH_DIR), getVersionTableName(version)), '/'); }
public String getBackupTableLocation() { Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.TRASH_DIR)); return StringUtils.join(Arrays.asList(getTrashDir(), getBackupTableName()), '/'); } }
private String getTrashTableLocation() { Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.TRASH_DIR), "Missing required property " + ComplianceConfigurationKeys.TRASH_DIR); return this.state.getProp(ComplianceConfigurationKeys.TRASH_DIR) + getCompleteTableName(); }
private static List<HiveDataset> getHiveDatasets(FileSystem fs, State state) throws IOException { Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.COMPLIANCE_DATASET_WHITELIST), "Missing required property " + ComplianceConfigurationKeys.COMPLIANCE_DATASET_WHITELIST); Properties prop = new Properties(); prop.setProperty(ComplianceConfigurationKeys.HIVE_DATASET_WHITELIST, state.getProp(ComplianceConfigurationKeys.COMPLIANCE_DATASET_WHITELIST)); HiveDatasetFinder finder = new HiveDatasetFinder(fs, prop); return finder.findDatasets(); }
private List<HivePartitionVersion> findVersions(String name, String urn) throws IOException { State state = new State(this.state); Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.HIVE_VERSIONS_WHITELIST), "Missing required property " + ComplianceConfigurationKeys.HIVE_VERSIONS_WHITELIST); state.setProp(ComplianceConfigurationKeys.HIVE_DATASET_WHITELIST, this.state.getProp(ComplianceConfigurationKeys.HIVE_VERSIONS_WHITELIST)); setVersions(name, state); log.info("Found " + this.versions.size() + " versions for the dataset " + urn); return this.versions; }
private DedupKeyOption getDedupKeyOption() { if (!this.dataset.jobProps().contains(COMPACTION_JOB_DEDUP_KEY)) { return DEFAULT_DEDUP_KEY_OPTION; } Optional<DedupKeyOption> option = Enums.getIfPresent(DedupKeyOption.class, this.dataset.jobProps().getProp(COMPACTION_JOB_DEDUP_KEY).toUpperCase()); return option.isPresent() ? option.get() : DEFAULT_DEDUP_KEY_OPTION; }
@Override public void updateSchema(HiveRegistrationUnit existingUnit, HiveRegistrationUnit newUnit) throws IOException { Preconditions.checkArgument( newUnit.getSerDeProps().contains(SCHEMA_LITERAL) || newUnit.getSerDeProps().contains(SCHEMA_URL)); if (newUnit.getSerDeProps().contains(SCHEMA_LITERAL)) { existingUnit.setSerDeProp(SCHEMA_LITERAL, newUnit.getSerDeProps().getProp(SCHEMA_LITERAL)); } else { existingUnit.setSerDeProp(SCHEMA_URL, newUnit.getSerDeProps().getProp(SCHEMA_URL)); } }
/** * Calls {@link #getOptionallyThrottledFileSystem(FileSystem, int)} parsing the qps from the input {@link State} * at key {@link #MAX_FILESYSTEM_QPS}. * @throws IOException */ public static FileSystem getOptionallyThrottledFileSystem(FileSystem fs, State state) throws IOException { DeprecationUtils.renameDeprecatedKeys(state, MAX_FILESYSTEM_QPS, DEPRECATED_KEYS); if (state.contains(MAX_FILESYSTEM_QPS)) { return getOptionallyThrottledFileSystem(fs, state.getPropAsInt(MAX_FILESYSTEM_QPS)); } return fs; }
private static void getFsAndJtTokensImpl(final State state, final Configuration conf, final Credentials cred) throws IOException { getHdfsToken(conf, cred); if (state.contains(OTHER_NAMENODES)) { getOtherNamenodesToken(state.getPropAsList(OTHER_NAMENODES), conf, cred); } getJtToken(cred); }