public RestApiConnector(State state) { this.state = state; this.authTokenTimeout = state.getPropAsInt(ConfigurationKeys.SOURCE_CONN_TIMEOUT, ConfigurationKeys.DEFAULT_CONN_TIMEOUT); }
/** * Get the number of {@link Dataset}s to be verified together. This allows multiple {@link Dataset}s * to share the same verification job, e.g., share the same query. */ private int getNumDatasetsVerifiedTogether() { return this.state.getPropAsInt(COMPACTION_COMPLETENESS_VERIFICATION_NUM_DATASETS_VERIFIED_TOGETHER, DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_NUM_DATASETS_VERIFIED_TOGETHER); }
public GenericJdbcBufferedInserter(State state, Connection conn) { super(state, conn); this.maxParamSize = state.getPropAsInt(WRITER_JDBC_MAX_PARAM_SIZE, DEFAULT_WRITER_JDBC_MAX_PARAM_SIZE); }
private int getDataCompletenessVerificationThreadPoolSize() { return this.props.getPropAsInt(COMPACTION_COMPLETENESS_VERIFICATION_THREAD_POOL_SIZE, DEFAULT_COMPACTION_COMPLETENESS_VERIFICATION_THREAD_POOL_SIZE); }
public MySqlBufferedInserter(State state, Connection conn) { super(state, conn); this.maxParamSize = state.getPropAsInt(WRITER_JDBC_MAX_PARAM_SIZE, DEFAULT_WRITER_JDBC_MAX_PARAM_SIZE); }
public static int getPartitionIdFromState(State s) { return s.getPropAsInt(PARTITION_ID); }
private int getThreadPoolSize() { return this.state.getPropAsInt(COMPACTION_THREAD_POOL_SIZE, DEFAULT_COMPACTION_THREAD_POOL_SIZE); }
/** * Constructor */ public PinotAuditCountHttpClient(State state) { int maxTotal = state.getPropAsInt(CONNECTION_MAX_TOTAL, DEFAULT_CONNECTION_MAX_TOTAL); int maxPerRoute = state.getPropAsInt(MAX_PER_ROUTE, DEFAULT_MAX_PER_ROUTE); cm = new PoolingHttpClientConnectionManager(); cm.setMaxTotal(maxTotal); cm.setDefaultMaxPerRoute(maxPerRoute); httpClient = HttpClients.custom() .setConnectionManager(cm) .build(); String host = state.getProp(TARGET_HOST); int port = state.getPropAsInt(TARGET_PORT); targetUrl = host + ":" + port + "/pql?pql="; }
public HivePartitionVersionRetentionReaperPolicy(State state, HivePartitionDataset dataset) { super(state, dataset); Preconditions.checkArgument(state.contains(ComplianceConfigurationKeys.REAPER_RETENTION_DAYS), "Missing required property " + ComplianceConfigurationKeys.REAPER_RETENTION_DAYS); this.retentionDays = state.getPropAsInt(ComplianceConfigurationKeys.REAPER_RETENTION_DAYS); }
private int getOwnFileCountThreshold (Dataset dataset) { int count = dataset.jobProps().getPropAsInt(MRCompactor.COMPACTION_LATEDATA_THRESHOLD_FILE_NUM, MRCompactor.DEFAULT_COMPACTION_LATEDATA_THRESHOLD_FILE_NUM); return count; }
public RecoveryHelper(FileSystem fs, State state) throws IOException { this.fs = fs; this.persistDir = getPersistDir(state); this.retentionHours = state.getPropAsInt(PERSIST_RETENTION_KEY, DEFAULT_PERSIST_RETENTION); }
public HivePartitionVersionRetentionCleanerPolicy(State state, HivePartitionDataset dataset) { super(state, dataset); Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_VERSIONS), "Missing required property " + ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_VERSIONS); Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_DAYS), "Missing required property " + ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_DAYS); Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.CLEANER_TRASH_RETENTION_DAYS), "Missing required property " + ComplianceConfigurationKeys.CLEANER_TRASH_RETENTION_DAYS); this.backupRetentionVersions = this.state.getPropAsInt(ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_VERSIONS); this.backupRetentionDays = this.state.getPropAsInt(ComplianceConfigurationKeys.CLEANER_BACKUP_RETENTION_DAYS); this.trashRetentionDays = this.state.getPropAsInt(ComplianceConfigurationKeys.CLEANER_TRASH_RETENTION_DAYS); }
private int getMaxNumReducers() { return this.dataset.jobProps().getPropAsInt(COMPACTION_JOB_MAX_NUM_REDUCERS, DEFAULT_COMPACTION_JOB_MAX_NUM_REDUCERS); }
GoogleWebmasterDataFetcherImpl(String siteProperty, GoogleWebmasterClient client, State wuState) throws IOException { _siteProperty = siteProperty; Preconditions.checkArgument(_siteProperty.endsWith("/"), "The site property must end in \"/\""); _client = client; _jobs = getHotStartJobs(wuState); API_REQUESTS_PER_SECOND = wuState.getPropAsDouble(GoogleWebMasterSource.KEY_PAGES_TUNING_REQUESTS_PER_SECOND, 4.5); PAGES_COUNT_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_COUNT_TUNING_COOLDOWN_TIME, 30); PAGES_GET_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_GET_TUNING_COOLDOWN_TIME, 5); LIMITER = new RateBasedLimiter(API_REQUESTS_PER_SECOND, TimeUnit.SECONDS); GET_PAGES_RETRIES = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_TUNING_MAX_RETRIES, 120); }
@VisibleForTesting GoogleDriveFsHelper(State state, Drive client, Closer closer) { this.closer = closer; this.fileSystem = this.closer.register(new GoogleDriveFileSystem(client)); if (state.contains(BUFFER_SIZE_BYTE)) { this.bufferSizeByte = Optional.of(state.getPropAsInt(BUFFER_SIZE_BYTE)); } else { this.bufferSizeByte = Optional.absent(); } }
public CachedInstanceKey(State state) { this.numOfEncryptionKeys = state.getPropAsInt(ConfigurationKeys.NUMBER_OF_ENCRYPT_KEYS, ConfigurationKeys.DEFAULT_NUMBER_OF_MASTER_PASSWORDS); this.useStrongEncryptor = shouldUseStrongEncryptor(state); this.fsURI = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC); this.masterPasswordFile = state.getProp(ConfigurationKeys.ENCRYPT_KEY_LOC); }
/** * @param state * @return Provides JdbcWriterCommands based on ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY */ public JdbcWriterCommands newInstance(State state, Connection conn) { String destKey = ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_DESTINATION_TYPE_KEY, state.getPropAsInt(ConfigurationKeys.FORK_BRANCHES_KEY, 1), state.getPropAsInt(ConfigurationKeys.FORK_BRANCH_ID_KEY, 0)); String destType = state.getProp(destKey); Preconditions.checkNotNull(destType, destKey + " is required for underlying JDBC product name"); return newInstance(Destination.of(DestinationType.valueOf(destType.toUpperCase()), state), conn); } }
public BaseJdbcBufferedInserter(State state, Connection conn) { this.conn = conn; this.batchSize = state.getPropAsInt(WRITER_JDBC_INSERT_BATCH_SIZE, DEFAULT_WRITER_JDBC_INSERT_BATCH_SIZE); if (this.batchSize < 1) { throw new IllegalArgumentException(WRITER_JDBC_INSERT_BATCH_SIZE + " should be a positive number"); } int maxWait = state.getPropAsInt(WRITER_JDBC_INSERT_RETRY_TIMEOUT, DEFAULT_WRITER_JDBC_INSERT_RETRY_TIMEOUT); int maxAttempts = state.getPropAsInt(WRITER_JDBC_INSERT_RETRY_MAX_ATTEMPT, DEFAULT_WRITER_JDBC_INSERT_RETRY_MAX_ATTEMPT); //retry after 2, 4, 8, 16... sec, allow at most maxWait sec delay this.retryer = RetryerBuilder.<Boolean> newBuilder().retryIfException() .withWaitStrategy(WaitStrategies.exponentialWait(1000, maxWait, TimeUnit.SECONDS)) .withStopStrategy(StopStrategies.stopAfterAttempt(maxAttempts)).build(); }
/** * Calls {@link #getOptionallyThrottledFileSystem(FileSystem, int)} parsing the qps from the input {@link State} * at key {@link #MAX_FILESYSTEM_QPS}. * @throws IOException */ public static FileSystem getOptionallyThrottledFileSystem(FileSystem fs, State state) throws IOException { DeprecationUtils.renameDeprecatedKeys(state, MAX_FILESYSTEM_QPS, DEPRECATED_KEYS); if (state.contains(MAX_FILESYSTEM_QPS)) { return getOptionallyThrottledFileSystem(fs, state.getPropAsInt(MAX_FILESYSTEM_QPS)); } return fs; }
public HiveAvroSerDeManager(State props) throws IOException { super(props); if (props.contains(HiveRegistrationPolicyBase.HIVE_FS_URI)) { this.fs = FileSystem.get(URI.create(props.getProp(HiveRegistrationPolicyBase.HIVE_FS_URI)), HadoopUtils.getConfFromState(props)); } else { this.fs = FileSystem.get(HadoopUtils.getConfFromState(props)); } this.useSchemaFile = props.getPropAsBoolean(USE_SCHEMA_FILE, DEFAULT_USE_SCHEMA_FILE); this.useSchemaTempFile = props.getPropAsBoolean(USE_SCHEMA_TEMP_FILE, DEFAULT_USE_SCHEMA_TEMP_FILE); this.schemaFileName = props.getProp(SCHEMA_FILE_NAME, DEFAULT_SCHEMA_FILE_NAME); this.schemaTempFileName = props.getProp(SCHEMA_TEMP_FILE_NAME, DEFAULT_SCHEMA_TEMP_FILE_NAME); this.schemaLiteralLengthLimit = props.getPropAsInt(SCHEMA_LITERAL_LENGTH_LIMIT, DEFAULT_SCHEMA_LITERAL_LENGTH_LIMIT); this.metricContext = Instrumented.getMetricContext(props, HiveAvroSerDeManager.class); }