private List<Dataset> prioritize (List<Dataset> datasets, State state) { double maxPool = state.getPropAsDouble(MRCompactor.COMPACTION_DATASETS_MAX_COUNT, MRCompactor.DEFUALT_COMPACTION_DATASETS_MAX_COUNT); ResourcePool pool = ResourcePool.builder().maxResource(SimpleDatasetRequest.SIMPLE_DATASET_COUNT_DIMENSION, maxPool).build(); Iterator<Dataset> newList = Iterators.transform( this.allocator.allocateRequests(datasets.stream().map(SimpleDatasetRequestor::new).iterator(), pool), (input) -> input.getDataset()); return Lists.newArrayList(newList); }
protected ResourcePool(ResourcePool other) { this.softBound = other.getSoftBound(); this.hardBound = other.getHardBound(); this.defaultResourceUse = other.getDefaultResourceUse(); this.dimensionIndex = other.getDimensionIndex(); }
public AllocatedRequestsIteratorBase(Iterator<RequestWithResourceRequirement<T>> underlying, ResourcePool resourcePool) { this.underlying = underlying; this.currentRequirement = new double[resourcePool.getNumDimensions()]; }
@Test public void test() { ResourcePool pool = ResourcePool.builder().maxResource(MEMORY, 1000.).maxResource(TIME, 200.).tolerance(MEMORY, 2.) .defaultRequirement(TIME, 1.).build(); Assert.assertEquals(pool.getNumDimensions(), 2); Assert.assertEquals(pool.getSoftBound(), new double[]{1000, 200}); // Default tolerance is 1.2 Assert.assertEquals(pool.getHardBound(), new double[]{2000, 240}); // Test default resource use Assert.assertEquals(pool.getResourceRequirementBuilder().build().getResourceVector(), new double[]{0, 1}); ResourceRequirement requirement = pool.getResourceRequirementBuilder().setRequirement(MEMORY, 10.).build(); Assert.assertEquals(requirement.getResourceVector(), new double[]{10, 1}); }
public ConcurrentBoundedPriorityIterable(final Comparator<? super T> prioritizer, ResourceEstimator<T> resourceEstimator, String storeRejectedRequestsSetting, ResourcePool pool) { this.estimator = resourceEstimator; this.resourcePool = pool; this.dimensions = this.resourcePool.getNumDimensions(); this.comparator = prioritizer; this.allDifferentComparator = new AllDifferentComparator(); this.elements = new TreeSet<>(this.allDifferentComparator); this.storeRejectedRequestsSetting = storeRejectedRequestsSetting; this.currentRequirement = this.resourcePool.getResourceRequirementBuilder().zero().build(); this.maxResourceRequirement = new ResourceRequirement(this.currentRequirement); this.candidateRequirement = new ResourceRequirement(this.currentRequirement); this.tmpRequirement = new ResourceRequirement(this.currentRequirement); this.reuse = new ResourceRequirement(this.currentRequirement); }
this.requestsOffered++; if (this.resourcePool.exceedsHardBound(newElement.getResourceRequirement(), false)) { this.resourcePool.stringifyRequirement(newElement.getResourceRequirement()))); if (!this.storeRejectedRequestsSetting .equalsIgnoreCase(RequestAllocatorConfig.StoreRejectedRequestsConfig.NONE.name())) { .add(this.currentRequirement, newElement.getResourceRequirement(), this.candidateRequirement); if (this.resourcePool.exceedsHardBound(candidateRequirement, false)) { if (!this.resourcePool.exceedsHardBound( ResourceRequirement.add(this.tmpRequirement, newElement.getResourceRequirement(), this.reuse), false)) { break;
@Override public ResourceRequirement estimateRequirement(String s, ResourcePool resourcePool) { double memory = Double.parseDouble(s.split("-")[1]); return resourcePool.getResourceRequirementBuilder().setRequirement(MEMORY, memory).build(); } }
@Override protected ResourcePool contractPool(ResourceRequirement requirement) { ResourcePool superPool = super.contractPool(requirement); return new CopyResourcePool(superPool); }
/** * @return a new {@link ResourcePool} which is a copy of this {@link ResourcePool} except its resource vector has been * reduced by the input {@link ResourceRequirement}. */ protected ResourcePool contractPool(ResourceRequirement requirement) { return new ResourcePool(VectorAlgebra.addVector(this.softBound, requirement.getResourceVector(), -1., null), VectorAlgebra.addVector(this.hardBound, requirement.getResourceVector(), -1., null), this.defaultResourceUse, this.dimensionIndex); }
/** * @return Whether the list has reached its soft bound. */ public synchronized boolean isFull() { return this.resourcePool.exceedsSoftBound(this.currentRequirement, true); }
public Builder(ResourcePool pool) { this.pool = pool; this.requirement = pool.getDefaultResourceUse(null); }
public ConcurrentBoundedPriorityIterable(final Comparator<? super T> prioritizer, ResourceEstimator<T> resourceEstimator, String storeRejectedRequestsSetting, ResourcePool pool) { this.estimator = resourceEstimator; this.resourcePool = pool; this.dimensions = this.resourcePool.getNumDimensions(); this.comparator = prioritizer; this.allDifferentComparator = new AllDifferentComparator(); this.elements = new TreeSet<>(this.allDifferentComparator); this.storeRejectedRequestsSetting = storeRejectedRequestsSetting; this.currentRequirement = this.resourcePool.getResourceRequirementBuilder().zero().build(); this.maxResourceRequirement = new ResourceRequirement(this.currentRequirement); this.candidateRequirement = new ResourceRequirement(this.currentRequirement); this.tmpRequirement = new ResourceRequirement(this.currentRequirement); this.reuse = new ResourceRequirement(this.currentRequirement); }
this.requestsOffered++; if (this.resourcePool.exceedsHardBound(newElement.getResourceRequirement(), false)) { this.resourcePool.stringifyRequirement(newElement.getResourceRequirement()))); if (!this.storeRejectedRequestsSetting .equalsIgnoreCase(RequestAllocatorConfig.StoreRejectedRequestsConfig.NONE.name())) { .add(this.currentRequirement, newElement.getResourceRequirement(), this.candidateRequirement); if (this.resourcePool.exceedsHardBound(candidateRequirement, false)) { if (!this.resourcePool.exceedsHardBound( ResourceRequirement.add(this.tmpRequirement, newElement.getResourceRequirement(), this.reuse), false)) { break;
@Override public ResourceRequirement estimateRequirement(StringRequest s, ResourcePool resourcePool) { double memory = Double.parseDouble(s.getString().split("-")[1]); this.queriedRequests.add(s.getString()); return resourcePool.getResourceRequirementBuilder().setRequirement(MEMORY, memory).build(); } }
@Override protected ResourcePool contractPool(ResourceRequirement requirement) { ResourcePool superPool = super.contractPool(requirement); return new CopyResourcePool(superPool); }
/** * @return a new {@link ResourcePool} which is a copy of this {@link ResourcePool} except its resource vector has been * reduced by the input {@link ResourceRequirement}. */ protected ResourcePool contractPool(ResourceRequirement requirement) { return new ResourcePool(VectorAlgebra.addVector(this.softBound, requirement.getResourceVector(), -1., null), VectorAlgebra.addVector(this.hardBound, requirement.getResourceVector(), -1., null), this.defaultResourceUse, this.dimensionIndex); }
/** * @return Whether the list has reached its soft bound. */ public synchronized boolean isFull() { return this.resourcePool.exceedsSoftBound(this.currentRequirement, true); }
public Builder(ResourcePool pool) { this.pool = pool; this.requirement = pool.getDefaultResourceUse(null); }
protected ResourcePool(ResourcePool other) { this.softBound = other.getSoftBound(); this.hardBound = other.getHardBound(); this.defaultResourceUse = other.getDefaultResourceUse(); this.dimensionIndex = other.getDimensionIndex(); }
@Test public void testAllocateRequests() throws Exception { RequestAllocatorConfig<StringRequest> configuration = RequestAllocatorConfig.builder(new StringRequest.StringRequestEstimator()).build(); GreedyAllocator<StringRequest> allocator = new GreedyAllocator<>(configuration); ResourcePool pool = ResourcePool.builder().maxResource(StringRequest.MEMORY, 100.).build(); List<Requestor<StringRequest>> requests = Lists.<Requestor<StringRequest>>newArrayList( new StringRequestor("r1", "a-50", "f-50", "k-20"), new StringRequestor("r2", "j-10", "b-20", "e-20"), new StringRequestor("r3", "g-20", "c-200", "d-30")); AllocatedRequestsIterator<StringRequest> result = allocator.allocateRequests(requests.iterator(), pool); List<StringRequest> resultList = Lists.newArrayList(result); Assert.assertEquals(resultList.size(), 2); // all equal, so no order guaranteed Assert.assertEquals(Sets.newHashSet(Lists.transform(resultList, new Function<StringRequest, String>() { @Override public String apply(StringRequest input) { return input.getString(); } })), Sets.newHashSet("a-50", "f-50")); }