private static double getPreGroupingSizeFactor(State state) { return state.getPropAsDouble(WORKUNIT_PRE_GROUPING_SIZE_FACTOR, DEFAULT_WORKUNIT_PRE_GROUPING_SIZE_FACTOR); }
/** * Constructor with user specified audit count client */ public CompactionAuditCountVerifier (State state, AuditCountClient client) { this.auditCountClient = client; this.state = state; // retrieve all tiers information if (client != null) { this.threshold = state.getPropAsDouble(COMPACTION_COMPLETENESS_THRESHOLD, DEFAULT_COMPACTION_COMPLETENESS_THRESHOLD); this.producerTier = state.getProp(PRODUCER_TIER); this.gobblinTier = state.getProp(GOBBLIN_TIER); this.originTiers = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(state.getProp(ORIGIN_TIER)); this.referenceTiers = new HashSet<>(originTiers); this.referenceTiers.add(producerTier); } }
private List<Dataset> prioritize (List<Dataset> datasets, State state) { double maxPool = state.getPropAsDouble(MRCompactor.COMPACTION_DATASETS_MAX_COUNT, MRCompactor.DEFUALT_COMPACTION_DATASETS_MAX_COUNT); ResourcePool pool = ResourcePool.builder().maxResource(SimpleDatasetRequest.SIMPLE_DATASET_COUNT_DIMENSION, maxPool).build(); Iterator<Dataset> newList = Iterators.transform( this.allocator.allocateRequests(datasets.stream().map(SimpleDatasetRequestor::new).iterator(), pool), (input) -> input.getDataset()); return Lists.newArrayList(newList); }
public RowCountRangePolicy(State state, Type type) { super(state, type); this.rowsRead = state.getPropAsLong(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED); this.rowsWritten = state.getPropAsLong(ConfigurationKeys.WRITER_ROWS_WRITTEN); this.range = state.getPropAsDouble(ConfigurationKeys.ROW_COUNT_RANGE); }
GoogleWebmasterDataFetcherImpl(String siteProperty, GoogleWebmasterClient client, State wuState) throws IOException { _siteProperty = siteProperty; Preconditions.checkArgument(_siteProperty.endsWith("/"), "The site property must end in \"/\""); _client = client; _jobs = getHotStartJobs(wuState); API_REQUESTS_PER_SECOND = wuState.getPropAsDouble(GoogleWebMasterSource.KEY_PAGES_TUNING_REQUESTS_PER_SECOND, 4.5); PAGES_COUNT_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_COUNT_TUNING_COOLDOWN_TIME, 30); PAGES_GET_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_GET_TUNING_COOLDOWN_TIME, 5); LIMITER = new RateBasedLimiter(API_REQUESTS_PER_SECOND, TimeUnit.SECONDS); GET_PAGES_RETRIES = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_TUNING_MAX_RETRIES, 120); }
/** * Get the average time to pull a record of a partition, which is stored in property * "[topicname].[partitionid].avg.record.millis". If state doesn't contain this property, it returns defaultValue. */ public static double getPartitionAvgRecordMillis(State state, KafkaPartition partition) { double avgRecordMillis = state.getPropAsDouble( getPartitionPropName(partition.getTopicName(), partition.getId()) + "." + KafkaSource.AVG_RECORD_MILLIS); // cap to prevent a poorly behaved topic from impacting the bin-packing int avgFetchTimeCap = state.getPropAsInt(ConfigurationKeys.KAFKA_SOURCE_AVG_FETCH_TIME_CAP, ConfigurationKeys.DEFAULT_KAFKA_SOURCE_AVG_FETCH_TIME_CAP); if (avgFetchTimeCap > 0 && avgRecordMillis > avgFetchTimeCap) { log.info("Topic {} partition {} has an average fetch time of {}, capping it to {}", partition.getTopicName(), partition.getId(), avgRecordMillis, avgFetchTimeCap); avgRecordMillis = avgFetchTimeCap; } return avgRecordMillis; }
Assert.assertEquals(state.getPropAsLong("long", Long.MAX_VALUE), Long.MAX_VALUE); Assert.assertEquals(state.getPropAsInt("int", Integer.MAX_VALUE), Integer.MAX_VALUE); Assert.assertEquals(state.getPropAsDouble("double", Double.MAX_VALUE), Double.MAX_VALUE); Assert.assertEquals(state.getPropAsBoolean("boolean", true), true); Assert.assertEquals(state.getPropAsLong("long"), Long.MAX_VALUE); Assert.assertEquals(state.getPropAsInt("int"), Integer.MAX_VALUE); Assert.assertEquals(state.getPropAsDouble("double"), Double.MAX_VALUE); Assert.assertEquals(state.getPropAsBoolean("boolean"), true); Assert.assertNotEquals(state.getPropAsLong("long", Long.MAX_VALUE), Long.MAX_VALUE); Assert.assertNotEquals(state.getPropAsInt("int", Integer.MAX_VALUE), Integer.MAX_VALUE); Assert.assertNotEquals(state.getPropAsDouble("double", Double.MAX_VALUE), Double.MAX_VALUE); Assert.assertNotEquals(state.getPropAsBoolean("boolean", true), true); Assert.assertNotEquals(state.getPropAsLong("long"), Long.MAX_VALUE); Assert.assertNotEquals(state.getPropAsInt("int"), Integer.MAX_VALUE); Assert.assertNotEquals(state.getPropAsDouble("double"), Double.MAX_VALUE); Assert.assertNotEquals(state.getPropAsBoolean("boolean"), true); Assert.assertEquals(state.getPropAsLong("long"), Long.MIN_VALUE); Assert.assertEquals(state.getPropAsInt("int"), Integer.MIN_VALUE); Assert.assertEquals(state.getPropAsDouble("double"), Double.MIN_VALUE); Assert.assertEquals(state.getPropAsBoolean("boolean"), false); Assert.assertEquals(state.getPropAsLong("long"), Long.MIN_VALUE); Assert.assertEquals(state.getPropAsInt("int"), Integer.MIN_VALUE); Assert.assertEquals(state.getPropAsDouble("double"), Double.MIN_VALUE); Assert.assertEquals(state.getPropAsBoolean("boolean"), false);
private static double getPreGroupingSizeFactor(State state) { return state.getPropAsDouble(WORKUNIT_PRE_GROUPING_SIZE_FACTOR, DEFAULT_WORKUNIT_PRE_GROUPING_SIZE_FACTOR); }
/** * Constructor with user specified audit count client */ public CompactionAuditCountVerifier (State state, AuditCountClient client) { this.auditCountClient = client; this.state = state; // retrieve all tiers information if (client != null) { this.threshold = state.getPropAsDouble(COMPACTION_COMPLETENESS_THRESHOLD, DEFAULT_COMPACTION_COMPLETENESS_THRESHOLD); this.producerTier = state.getProp(PRODUCER_TIER); this.gobblinTier = state.getProp(GOBBLIN_TIER); this.originTiers = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(state.getProp(ORIGIN_TIER)); this.referenceTiers = new HashSet<>(originTiers); this.referenceTiers.add(producerTier); } }
public RowCountRangePolicy(State state, Type type) { super(state, type); this.rowsRead = state.getPropAsLong(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED); this.rowsWritten = state.getPropAsLong(ConfigurationKeys.WRITER_ROWS_WRITTEN); this.range = state.getPropAsDouble(ConfigurationKeys.ROW_COUNT_RANGE); }
private List<Dataset> prioritize (List<Dataset> datasets, State state) { double maxPool = state.getPropAsDouble(MRCompactor.COMPACTION_DATASETS_MAX_COUNT, MRCompactor.DEFUALT_COMPACTION_DATASETS_MAX_COUNT); ResourcePool pool = ResourcePool.builder().maxResource(SimpleDatasetRequest.SIMPLE_DATASET_COUNT_DIMENSION, maxPool).build(); Iterator<Dataset> newList = Iterators.transform( this.allocator.allocateRequests(datasets.stream().map(SimpleDatasetRequestor::new).iterator(), pool), (input) -> input.getDataset()); return Lists.newArrayList(newList); }
GoogleWebmasterDataFetcherImpl(String siteProperty, GoogleWebmasterClient client, State wuState) throws IOException { _siteProperty = siteProperty; Preconditions.checkArgument(_siteProperty.endsWith("/"), "The site property must end in \"/\""); _client = client; _jobs = getHotStartJobs(wuState); API_REQUESTS_PER_SECOND = wuState.getPropAsDouble(GoogleWebMasterSource.KEY_PAGES_TUNING_REQUESTS_PER_SECOND, 4.5); PAGES_COUNT_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_COUNT_TUNING_COOLDOWN_TIME, 30); PAGES_GET_COOLDOWN_TIME = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_GET_TUNING_COOLDOWN_TIME, 5); LIMITER = new RateBasedLimiter(API_REQUESTS_PER_SECOND, TimeUnit.SECONDS); GET_PAGES_RETRIES = wuState.getPropAsInt(GoogleWebMasterSource.KEY_PAGES_TUNING_MAX_RETRIES, 120); }
/** * Get the average time to pull a record of a partition, which is stored in property * "[topicname].[partitionid].avg.record.millis". If state doesn't contain this property, it returns defaultValue. */ public static double getPartitionAvgRecordMillis(State state, KafkaPartition partition) { double avgRecordMillis = state.getPropAsDouble( getPartitionPropName(partition.getTopicName(), partition.getId()) + "." + KafkaSource.AVG_RECORD_MILLIS); // cap to prevent a poorly behaved topic from impacting the bin-packing int avgFetchTimeCap = state.getPropAsInt(ConfigurationKeys.KAFKA_SOURCE_AVG_FETCH_TIME_CAP, ConfigurationKeys.DEFAULT_KAFKA_SOURCE_AVG_FETCH_TIME_CAP); if (avgFetchTimeCap > 0 && avgRecordMillis > avgFetchTimeCap) { log.info("Topic {} partition {} has an average fetch time of {}, capping it to {}", partition.getTopicName(), partition.getId(), avgRecordMillis, avgFetchTimeCap); avgRecordMillis = avgFetchTimeCap; } return avgRecordMillis; }