private RequestAllocator<FileSet<CopyEntity>> createRequestAllocator(CopyConfiguration copyConfiguration, int maxThreads) { Optional<FileSetComparator> prioritizer = copyConfiguration.getPrioritizer(); RequestAllocatorConfig.Builder<FileSet<CopyEntity>> configBuilder = RequestAllocatorConfig.builder(new FileSetResourceEstimator()).allowParallelization(maxThreads) .storeRejectedRequests(copyConfiguration.getStoreRejectedRequestsSetting()) .withLimitedScopeConfig(copyConfiguration.getPrioritizationConfig()); if (!prioritizer.isPresent()) { return new GreedyAllocator<>(configBuilder.build()); } else { configBuilder.withPrioritizer(prioritizer.get()); } if (prioritizer.get() instanceof HierarchicalPrioritizer) { return new HierarchicalAllocator.Factory().createRequestAllocator(configBuilder.build()); } else { return RequestAllocatorUtils.inferFromConfig(configBuilder.build()); } }
private void initRequestAllocator (State state) { try { ResourceEstimator estimator = GobblinConstructorUtils.<ResourceEstimator>invokeLongestConstructor( new ClassAliasResolver(ResourceEstimator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_ESTIMATOR, SimpleDatasetRequest.SimpleDatasetCountEstimator.class.getName()))); RequestAllocatorConfig.Builder<SimpleDatasetRequest> configBuilder = RequestAllocatorConfig.builder(estimator).allowParallelization(1).withLimitedScopeConfig(ConfigBuilder.create() .loadProps(state.getProperties(), ConfigurationKeys.COMPACTION_PRIORITIZATION_PREFIX).build()); if (!state.contains(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)) { allocator = new GreedyAllocator<>(configBuilder.build()); return; } Comparator<SimpleDatasetRequest> prioritizer = GobblinConstructorUtils.<Comparator>invokeLongestConstructor( new ClassAliasResolver(Comparator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)), state); configBuilder.withPrioritizer(prioritizer); if (prioritizer instanceof HierarchicalPrioritizer) { allocator = new HierarchicalAllocator.Factory().createRequestAllocator(configBuilder.build()); } else { allocator = RequestAllocatorUtils.inferFromConfig(configBuilder.build()); } } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException("Cannot initialize allocator", e); } }
private RequestAllocator<FileSet<CopyEntity>> createRequestAllocator(CopyConfiguration copyConfiguration, int maxThreads) { Optional<FileSetComparator> prioritizer = copyConfiguration.getPrioritizer(); RequestAllocatorConfig.Builder<FileSet<CopyEntity>> configBuilder = RequestAllocatorConfig.builder(new FileSetResourceEstimator()).allowParallelization(maxThreads) .storeRejectedRequests(copyConfiguration.getStoreRejectedRequestsSetting()) .withLimitedScopeConfig(copyConfiguration.getPrioritizationConfig()); if (!prioritizer.isPresent()) { return new GreedyAllocator<>(configBuilder.build()); } else { configBuilder.withPrioritizer(prioritizer.get()); } if (prioritizer.get() instanceof HierarchicalPrioritizer) { return new HierarchicalAllocator.Factory().createRequestAllocator(configBuilder.build()); } else { return RequestAllocatorUtils.inferFromConfig(configBuilder.build()); } }
public static <T extends Request<T>> Builder<T> builder(ResourceEstimator<T> resourceEstimator) { return new Builder<>(resourceEstimator); }
public Builder<T> allowParallelization() { return allowParallelization(20); }
new SimpleHierarchicalPrioritizer<>(requestorComparator, new StringRequest.StringRequestComparator()); RequestAllocatorConfig<StringRequest> configuration = RequestAllocatorConfig.builder(new StringRequest.StringRequestEstimator()).withPrioritizer(prioritizer).build(); BruteForceAllocator<StringRequest> underlying = new BruteForceAllocator<>(configuration); HierarchicalAllocator<StringRequest> hierarchicalAllocator = new HierarchicalAllocator<>(prioritizer, underlying);
@Test public void testAllocateRequests() throws Exception { StringRequest.StringRequestEstimator estimator = new StringRequest.StringRequestEstimator(); RequestAllocatorConfig<StringRequest> configuration = RequestAllocatorConfig.builder(estimator).withPrioritizer(new StringRequest.StringRequestComparator()).build(); PreOrderAllocator<StringRequest> allocator = new PreOrderAllocator<>(configuration); ResourcePool pool = ResourcePool.builder().maxResource(StringRequest.MEMORY, 100.).build(); List<Requestor<StringRequest>> requests = Lists.<Requestor<StringRequest>>newArrayList( new StringRequestor("r1", "a-50", "f-50", "k-20"), new StringRequestor("r2", "j-10", "b-20", "e-20"), new StringRequestor("r3", "g-20", "c-200", "d-30")); AllocatedRequestsIterator<StringRequest> result = allocator.allocateRequests(requests.iterator(), pool); List<StringRequest> resultList = Lists.newArrayList(result); Assert.assertEquals(resultList.size(), 3); Assert.assertEquals(resultList.get(0).getString(), "a-50"); Assert.assertEquals(resultList.get(1).getString(), "b-20"); // No c because it is too large to fit Assert.assertEquals(resultList.get(2).getString(), "d-30"); Assert.assertTrue(estimator.getQueriedRequests().contains("a-50")); Assert.assertTrue(estimator.getQueriedRequests().contains("b-20")); Assert.assertTrue(estimator.getQueriedRequests().contains("c-200")); Assert.assertTrue(estimator.getQueriedRequests().contains("d-30")); Assert.assertFalse(estimator.getQueriedRequests().contains("e-20")); Assert.assertFalse(estimator.getQueriedRequests().contains("f-50")); }
private void initRequestAllocator (State state) { try { ResourceEstimator estimator = GobblinConstructorUtils.<ResourceEstimator>invokeLongestConstructor( new ClassAliasResolver(ResourceEstimator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_ESTIMATOR, SimpleDatasetRequest.SimpleDatasetCountEstimator.class.getName()))); RequestAllocatorConfig.Builder<SimpleDatasetRequest> configBuilder = RequestAllocatorConfig.builder(estimator).allowParallelization(1).withLimitedScopeConfig(ConfigBuilder.create() .loadProps(state.getProperties(), ConfigurationKeys.COMPACTION_PRIORITIZATION_PREFIX).build()); if (!state.contains(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)) { allocator = new GreedyAllocator<>(configBuilder.build()); return; } Comparator<SimpleDatasetRequest> prioritizer = GobblinConstructorUtils.<Comparator>invokeLongestConstructor( new ClassAliasResolver(Comparator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)), state); configBuilder.withPrioritizer(prioritizer); if (prioritizer instanceof HierarchicalPrioritizer) { allocator = new HierarchicalAllocator.Factory().createRequestAllocator(configBuilder.build()); } else { allocator = RequestAllocatorUtils.inferFromConfig(configBuilder.build()); } } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException("Cannot initialize allocator", e); } }
@Test public void testAllocateRequests() throws Exception { RequestAllocatorConfig<StringRequest> configuration = RequestAllocatorConfig.builder(new StringRequest.StringRequestEstimator()) .withPrioritizer(new StringRequest.StringRequestComparator()).build(); BruteForceAllocator<StringRequest> allocator = new BruteForceAllocator<>(configuration); ResourcePool pool = ResourcePool.builder().maxResource(StringRequest.MEMORY, 100.).build(); List<Requestor<StringRequest>> requests = Lists.<Requestor<StringRequest>>newArrayList( new StringRequestor("r1", "a-50", "f-50", "k-20"), new StringRequestor("r2", "j-10", "b-20", "e-20"), new StringRequestor("r3", "g-20", "c-200", "d-30")); AllocatedRequestsIterator<StringRequest> result = allocator.allocateRequests(requests.iterator(), pool); List<StringRequest> resultList = Lists.newArrayList(result); Assert.assertEquals(resultList.size(), 4); Assert.assertEquals(resultList.get(0).getString(), "a-50"); Assert.assertEquals(resultList.get(1).getString(), "b-20"); // No c because it is too large to fit Assert.assertEquals(resultList.get(2).getString(), "d-30"); Assert.assertEquals(resultList.get(3).getString(), "e-20"); } }
@Test public void testAllocateRequests() throws Exception { RequestAllocatorConfig<StringRequest> configuration = RequestAllocatorConfig.builder(new StringRequest.StringRequestEstimator()).build(); GreedyAllocator<StringRequest> allocator = new GreedyAllocator<>(configuration); ResourcePool pool = ResourcePool.builder().maxResource(StringRequest.MEMORY, 100.).build(); List<Requestor<StringRequest>> requests = Lists.<Requestor<StringRequest>>newArrayList( new StringRequestor("r1", "a-50", "f-50", "k-20"), new StringRequestor("r2", "j-10", "b-20", "e-20"), new StringRequestor("r3", "g-20", "c-200", "d-30")); AllocatedRequestsIterator<StringRequest> result = allocator.allocateRequests(requests.iterator(), pool); List<StringRequest> resultList = Lists.newArrayList(result); Assert.assertEquals(resultList.size(), 2); // all equal, so no order guaranteed Assert.assertEquals(Sets.newHashSet(Lists.transform(resultList, new Function<StringRequest, String>() { @Override public String apply(StringRequest input) { return input.getString(); } })), Sets.newHashSet("a-50", "f-50")); }
public Builder<T> allowParallelization() { return allowParallelization(20); }
public static <T extends Request<T>> Builder<T> builder(ResourceEstimator<T> resourceEstimator) { return new Builder<>(resourceEstimator); }