@Override public String datasetURN() { return state.getProp(SERIALIZE_COMPACTION_FILE_PATH_NAME); } };
public CachedInstanceKey(State state) { this.numOfEncryptionKeys = state.getPropAsInt(ConfigurationKeys.NUMBER_OF_ENCRYPT_KEYS, ConfigurationKeys.DEFAULT_NUMBER_OF_MASTER_PASSWORDS); this.useStrongEncryptor = shouldUseStrongEncryptor(state); this.fsURI = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC); this.masterPasswordFile = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC); }
public HiveBaseExtractor(WorkUnitState state) throws IOException { if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) { return; } this.hiveWorkUnit = new HiveWorkUnit(state.getWorkunit()); this.hiveDataset = hiveWorkUnit.getHiveDataset(); this.dbName = hiveDataset.getDbAndTable().getDb(); this.tableName = hiveDataset.getDbAndTable().getTable(); this.pool = HiveMetastoreClientPool.get(state.getJobState().getProperties(), Optional.fromNullable(state.getJobState().getProp(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); }
/** * Check whether metrics collection and reporting are enabled or not. * * @param state a {@link State} object containing configuration properties * @return whether metrics collection and reporting are enabled */ public static boolean isEnabled(State state) { return Boolean .valueOf(state.getProp(ConfigurationKeys.METRICS_ENABLED_KEY, ConfigurationKeys.DEFAULT_METRICS_ENABLED)); }
/** * Get the value of a property as an short, using the given default value if the property is not set. * * @param key property key * @param def default value * @return short value associated with the key or the default value if the property is not set */ public short getPropAsShort(String key, short def) { return Short.parseShort(getProp(key, String.valueOf(def))); }
/** * Get the value of a property as a double, using the given default value if the property is not set. * * @param key property key * @param def default value * @return double value associated with the key or the default value if the property is not set */ public double getPropAsDouble(String key, double def) { return Double.parseDouble(getProp(key, String.valueOf(def))); }
/** * @return the {@link Split} object contained in the {@link WorkUnit}. */ public static Optional<Split> getSplit(State workUnit) { return workUnit.contains(SPLIT_KEY) ? Optional.of(GSON.fromJson(workUnit.getProp(SPLIT_KEY), Split.class)) : Optional.<Split>absent(); }
private String getDstBaseDir(State state) { Preconditions.checkArgument(state.contains(MRCompactor.COMPACTION_DEST_DIR), "Missing required property " + MRCompactor.COMPACTION_DEST_DIR); return state.getProp(MRCompactor.COMPACTION_DEST_DIR); }
private String getDstSubDir(State state) { Preconditions.checkArgument(state.contains(MRCompactor.COMPACTION_DEST_SUBDIR), "Missing required property " + MRCompactor.COMPACTION_DEST_SUBDIR); return state.getProp(MRCompactor.COMPACTION_DEST_SUBDIR); } }
/** * Read execution count from a specific directory. * File name is {@link InputRecordCountHelper#STATE_FILE} * @param dir directory where a state file is located * @return record count */ public long readExecutionCount (Path dir) throws IOException { State state = loadState(fs, dir); return Long.parseLong(state.getProp(CompactionSlaEventHelper.EXEC_COUNT_TOTAL, "0")); }
/** * Get the full lineage event name from a state */ public static String getFullEventName(State state) { return Joiner.on('.').join(LineageEventBuilder.LIENAGE_EVENT_NAMESPACE, state.getProp(getKey(NAME_KEY))); }
private FsRenameCommitStep(Builder<? extends Builder<?>> builder) throws IOException { super(builder); this.srcPath = builder.srcPath; this.dstPath = builder.dstPath; this.srcFs = builder.srcFs != null ? builder.srcFs : getFileSystem(this.props.getProp(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI)); this.srcFsUri = this.srcFs.getUri().toString(); this.dstFs = builder.dstFs != null ? builder.dstFs : getFileSystem(this.props.getProp(ConfigurationKeys.FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI)); this.dstFsUri = this.dstFs.getUri().toString(); this.overwrite = builder.overwrite; }
private HiveConnection getHiveConnection(Optional<String> proxyUser) throws ClassNotFoundException, SQLException { Class.forName("org.apache.hive.jdbc.HiveDriver"); Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.HIVE_JDBC_URL), "Missing required property " + ComplianceConfigurationKeys.HIVE_JDBC_URL); String url = this.state.getProp(ComplianceConfigurationKeys.HIVE_JDBC_URL); if (proxyUser.isPresent()) { url = url + ComplianceConfigurationKeys.HIVE_SERVER2_PROXY_USER + proxyUser.get(); } return (HiveConnection) DriverManager.getConnection(url); }
private String getBackUpTableLocation(HivePartitionVersion version) { Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.TRASH_DIR), "Missing required property " + ComplianceConfigurationKeys.TRASH_DIR); return StringUtils .join(Arrays.asList(this.state.getProp(ComplianceConfigurationKeys.TRASH_DIR), getVersionTableName(version)), '/'); }
private static DatePartitionType getGranularity(State state, int numBranches, int branchId) { String propName = ForkOperatorUtils.getPropertyNameForBranch(WRITER_PARTITION_GRANULARITY, numBranches, branchId); String granularityValue = state.getProp(propName, DEFAULT_WRITER_PARTITION_GRANULARITY.toString()); Optional<DatePartitionType> granularity = Enums.getIfPresent(DatePartitionType.class, granularityValue.toUpperCase()); Preconditions.checkState(granularity.isPresent(), granularityValue + " is not a valid writer partition granularity"); return granularity.get(); }
private CombineOperation getConditionOperation (Dataset dataset) { String oprName = dataset.jobProps().getProp (MRCompactor.COMPACTION_RECOMPACT_COMBINE_CONDITIONS_OPERATION, MRCompactor.DEFAULT_COMPACTION_RECOMPACT_COMBINE_CONDITIONS_OPERATION); try { CombineOperation opr = CombineOperation.valueOf (oprName.toUpperCase()); return opr; } catch (Exception e) { return CombineOperation.OR; } }
private static List<HiveDataset> getHiveDatasets(FileSystem fs, State state) throws IOException { Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.COMPLIANCE_DATASET_WHITELIST), "Missing required property " + ComplianceConfigurationKeys.COMPLIANCE_DATASET_WHITELIST); Properties prop = new Properties(); prop.setProperty(ComplianceConfigurationKeys.HIVE_DATASET_WHITELIST, state.getProp(ComplianceConfigurationKeys.COMPLIANCE_DATASET_WHITELIST)); HiveDatasetFinder finder = new HiveDatasetFinder(fs, prop); return finder.findDatasets(); }
@VisibleForTesting public Connection createConnection() throws SQLException { DataSource dataSource = DataSourceBuilder.builder().url(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_URL)) .driver(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_DRIVER)) .userName(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_USERNAME)) .passWord(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_PASSWORD)) .cryptoKeyLocation(this.state.getProp(JdbcPublisher.JDBC_PUBLISHER_ENCRYPTION_KEY_LOC)).maxActiveConnections(1) .maxIdleConnections(1).state(this.state).build(); return dataSource.getConnection(); }
private DateTime getEarliestAllowedFolderTime(DateTime currentTime, PeriodFormatter periodFormatter) { String maxTimeAgoStr = this.state.getProp(COMPACTION_TIMEBASED_MAX_TIME_AGO, DEFAULT_COMPACTION_TIMEBASED_MAX_TIME_AGO); Period maxTimeAgo = periodFormatter.parsePeriod(maxTimeAgoStr); return currentTime.minus(maxTimeAgo); }