AsyncHttpWriterBuilder<D, RQ, RP> fromState(State state) { if (!(state instanceof WorkUnitState)) { throw new IllegalStateException(String.format("AsyncHttpWriterBuilder requires a %s on construction.", WorkUnitState.class.getSimpleName())); } this.state = (WorkUnitState) state; this.metricContext = Instrumented.getMetricContext(this.state, AsyncHttpWriter.class); this.broker = this.state.getTaskBroker(); Config config = ConfigBuilder.create().loadProps(state.getProperties(), CONF_PREFIX).build(); config = config.withFallback(FALLBACK); this.maxOutstandingWrites = config.getInt(MAX_OUTSTANDING_WRITES); this.maxAttempts = config.getInt(MAX_ATTEMPTS); return fromConfig(config); }
.loadProps(state.getProperties(), POLL_RETRY_PREFIX) .build() .withFallback(POLL_RETRY_DEFAULTS);
public B fromState(State state) { this.state = state; Config config = ConfigBuilder.create().loadProps(state.getProperties(), CONF_PREFIX).build(); fromConfig(config); return typedSelf(); }
public HttpJoinConverter init(WorkUnitState workUnitState) { super.init(workUnitState); Config config = ConfigBuilder.create().loadProps(workUnitState.getProperties(), CONF_PREFIX).build(); config = config.withFallback(DEFAULT_FALLBACK); httpClient = createHttpClient(config, workUnitState.getTaskBroker()); responseHandler = createResponseHandler(config); requestBuilder = createRequestBuilder(config); return this; }
public AsyncHttpJoinConverter init(WorkUnitState workUnitState) { super.init(workUnitState); Config config = ConfigBuilder.create().loadProps(workUnitState.getProperties(), CONF_PREFIX).build(); config = config.withFallback(DEFAULT_FALLBACK); skipFailedRecord = workUnitState.getPropAsBoolean(ConfigurationKeys.CONVERTER_SKIP_FAILED_RECORD, false); httpClient = createHttpClient(config, workUnitState.getTaskBroker()); responseHandler = createResponseHandler(config); requestBuilder = createRequestBuilder(config); return this; }
.loadProps(this.getState().getProperties(), DATA_PUBLISHER_RETRY_PREFIX) .build() .withFallback(PUBLISH_RETRY_DEFAULTS);
.insert(accountId, webPropertyId, profileId, request); Config config = ConfigBuilder.create().loadProps(wuState.getProperties(), REQUEST_RETRY_PREFIX).build(); Retryer<UnsampledReport> retryer = RetryerFactory.newInstance(config);
.loadProps(this.dataset.jobProps().getProperties(), COMPACTION_RETRY_PREFIX) .build() .withFallback(COMPACTION_RETRY_DEFAULTS);
private void initRequestAllocator (State state) { try { ResourceEstimator estimator = GobblinConstructorUtils.<ResourceEstimator>invokeLongestConstructor( new ClassAliasResolver(ResourceEstimator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_ESTIMATOR, SimpleDatasetRequest.SimpleDatasetCountEstimator.class.getName()))); RequestAllocatorConfig.Builder<SimpleDatasetRequest> configBuilder = RequestAllocatorConfig.builder(estimator).allowParallelization(1).withLimitedScopeConfig(ConfigBuilder.create() .loadProps(state.getProperties(), ConfigurationKeys.COMPACTION_PRIORITIZATION_PREFIX).build()); if (!state.contains(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)) { allocator = new GreedyAllocator<>(configBuilder.build()); return; } Comparator<SimpleDatasetRequest> prioritizer = GobblinConstructorUtils.<Comparator>invokeLongestConstructor( new ClassAliasResolver(Comparator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)), state); configBuilder.withPrioritizer(prioritizer); if (prioritizer instanceof HierarchicalPrioritizer) { allocator = new HierarchicalAllocator.Factory().createRequestAllocator(configBuilder.build()); } else { allocator = RequestAllocatorUtils.inferFromConfig(configBuilder.build()); } } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException("Cannot initialize allocator", e); } }
AsyncHttpWriterBuilder<D, RQ, RP> fromState(State state) { if (!(state instanceof WorkUnitState)) { throw new IllegalStateException(String.format("AsyncHttpWriterBuilder requires a %s on construction.", WorkUnitState.class.getSimpleName())); } this.state = (WorkUnitState) state; this.metricContext = Instrumented.getMetricContext(this.state, AsyncHttpWriter.class); this.broker = this.state.getTaskBroker(); Config config = ConfigBuilder.create().loadProps(state.getProperties(), CONF_PREFIX).build(); config = config.withFallback(FALLBACK); this.maxOutstandingWrites = config.getInt(MAX_OUTSTANDING_WRITES); this.maxAttempts = config.getInt(MAX_ATTEMPTS); return fromConfig(config); }
.loadProps(state.getProperties(), POLL_RETRY_PREFIX) .build() .withFallback(POLL_RETRY_DEFAULTS);
public B fromState(State state) { this.state = state; Config config = ConfigBuilder.create().loadProps(state.getProperties(), CONF_PREFIX).build(); fromConfig(config); return typedSelf(); }
public HttpJoinConverter init(WorkUnitState workUnitState) { super.init(workUnitState); Config config = ConfigBuilder.create().loadProps(workUnitState.getProperties(), CONF_PREFIX).build(); config = config.withFallback(DEFAULT_FALLBACK); httpClient = createHttpClient(config, workUnitState.getTaskBroker()); responseHandler = createResponseHandler(config); requestBuilder = createRequestBuilder(config); return this; }
public AsyncHttpJoinConverter init(WorkUnitState workUnitState) { super.init(workUnitState); Config config = ConfigBuilder.create().loadProps(workUnitState.getProperties(), CONF_PREFIX).build(); config = config.withFallback(DEFAULT_FALLBACK); skipFailedRecord = workUnitState.getPropAsBoolean(ConfigurationKeys.CONVERTER_SKIP_FAILED_RECORD, false); httpClient = createHttpClient(config, workUnitState.getTaskBroker()); responseHandler = createResponseHandler(config); requestBuilder = createRequestBuilder(config); return this; }
.loadProps(this.getState().getProperties(), DATA_PUBLISHER_RETRY_PREFIX) .build() .withFallback(PUBLISH_RETRY_DEFAULTS);
.insert(accountId, webPropertyId, profileId, request); Config config = ConfigBuilder.create().loadProps(wuState.getProperties(), REQUEST_RETRY_PREFIX).build(); Retryer<UnsampledReport> retryer = RetryerFactory.newInstance(config);
.loadProps(this.dataset.jobProps().getProperties(), COMPACTION_RETRY_PREFIX) .build() .withFallback(COMPACTION_RETRY_DEFAULTS);
private void initRequestAllocator (State state) { try { ResourceEstimator estimator = GobblinConstructorUtils.<ResourceEstimator>invokeLongestConstructor( new ClassAliasResolver(ResourceEstimator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_ESTIMATOR, SimpleDatasetRequest.SimpleDatasetCountEstimator.class.getName()))); RequestAllocatorConfig.Builder<SimpleDatasetRequest> configBuilder = RequestAllocatorConfig.builder(estimator).allowParallelization(1).withLimitedScopeConfig(ConfigBuilder.create() .loadProps(state.getProperties(), ConfigurationKeys.COMPACTION_PRIORITIZATION_PREFIX).build()); if (!state.contains(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)) { allocator = new GreedyAllocator<>(configBuilder.build()); return; } Comparator<SimpleDatasetRequest> prioritizer = GobblinConstructorUtils.<Comparator>invokeLongestConstructor( new ClassAliasResolver(Comparator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)), state); configBuilder.withPrioritizer(prioritizer); if (prioritizer instanceof HierarchicalPrioritizer) { allocator = new HierarchicalAllocator.Factory().createRequestAllocator(configBuilder.build()); } else { allocator = RequestAllocatorUtils.inferFromConfig(configBuilder.build()); } } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException("Cannot initialize allocator", e); } }