/** * Creates the <i>power list</i> of the given {@code base} (akin to power sets of sets). * * @param maxElements maximum number of elements in the {@link List}s in the power list */ public static <T> Collection<List<T>> createPowerList(Collection<T> base, int maxElements) { List<T> baseList = asList(base); List<List<T>> powerList = new ArrayList<>(); createPowerListAux(baseList, 0, maxElements, powerList); powerList.sort((a, b) -> Integer.compare(a.size(), b.size())); return powerList; }
@Override public void open(ExecutionContext ctx) { long numVertices = RheemCollections.getSingle(ctx.getBroadcast("numVertices")); this.minRank = (1 - this.dampingFactor) / numVertices; }
/** * Creates a new instance. * * @param inputSlots see {@link #inputSlots} * @param borderInputSlots see {@link #borderInputSlots} * @param sourceOperators see {@link #sourceOperators} * @param configuration see {@link #configuration} */ private Builder(Collection<InputSlot<?>> inputSlots, Collection<InputSlot<?>> borderInputSlots, Collection<Operator> sourceOperators, Configuration configuration) { this.inputSlots = inputSlots; this.borderInputSlots = RheemCollections.asSet(borderInputSlots); this.configuration = configuration; this.sourceOperators = sourceOperators; }
List<InputSlot<?>> emptyGroupInputs = RheemCollections.createNullFilledArrayList(inputs.size()); for (PlanImplementation planImplementation : this.getPlanImplementations()) { PlanImplementation.ConcatenationDescriptor concatDescriptor = orderedConcatGroups.add(enum2concatGroup.get(enumeration)); for (List<PlanImplementation.ConcatenationGroupDescriptor> concatGroupCombo : RheemCollections.streamedCrossProduct(orderedConcatGroups)) { final PlanImplementation innerPlanImplementation = RheemCollections.getAny(baseConcatDescriptors).execOutputPlanImplementation; final List<Set<PlanImplementation>> groupPlans = RheemCollections.map( concatGroupCombo, concatGroup -> { }); for (List<PlanImplementation> planCombo : RheemCollections.streamedCrossProduct(groupPlans)) { PlanImplementation basePlan = planCombo.get(0); List<PlanImplementation> targetPlans = planCombo.subList(0, planCombo.size());
/** * Creates a new instance. */ ConcatenationDescriptor(OutputSlot<?> output, List<InputSlot<?>> inputs) { // Find the ExecutionOperator's corresponding OutputSlot along with the nested PlanImplementation. OutputSlot<?> execOutput = null; PlanImplementation execOutputPlanImplementation = null; if (output != null) { Collection<Tuple<OutputSlot<?>, PlanImplementation>> execOpOutputsWithContext = PlanImplementation.this.findExecutionOperatorOutputWithContext(output); final Tuple<OutputSlot<?>, PlanImplementation> execOpOutputWithCtx = RheemCollections.getSingleOrNull(execOpOutputsWithContext); assert execOpOutputsWithContext != null : String.format("No outputs found for %s.", output); execOutput = execOpOutputWithCtx.field0; execOutputPlanImplementation = execOpOutputWithCtx.field1; } // Find the ExecutionOperators' corresponding InputSlots. List<Set<InputSlot<?>>> execInputs = new ArrayList<>(inputs.size()); for (InputSlot<?> input : inputs) { if (input == null) { execInputs.add(null); } else { execInputs.add(RheemCollections.asSet(PlanImplementation.this.findExecutionOperatorInputs(input))); } } this.groupDescriptor = new ConcatenationGroupDescriptor(execOutput, execInputs); this.execOutputPlanImplementation = execOutputPlanImplementation; }
/** * Creates the final {@link PlanEnumeration} by <ol> * <li>escaping all terminal operations (in {@link #completedEnumerations}) from {@link #enumeratedAlternative} and</li> * <li>joining them.</li> * </ol> * The result is stored in {@link #resultReference}. Note the outcome might be {@code null} if the traversed plan * did not allow to construct a valid {@link PlanEnumeration}. */ private void constructResultEnumeration() { final PlanEnumeration resultEnumeration = RheemCollections.getSingleOrNull(this.completedEnumerations); this.resultReference = new AtomicReference<>(resultEnumeration); }
Channel existingChannel = RheemCollections.getAny(openChannels); while (existingChannel.getProducerSlot() != sourceOutput) { existingChannel = OptimizationUtils.getPredecessorChannel(existingChannel); this.destChannelDescriptorSets = RheemCollections.map(destInputs, this::resolveSupportedChannels); assert this.destChannelDescriptorSets.stream().noneMatch(Collection::isEmpty); this.kernelizeChannelRequests();
RheemCollections.createPowerList(childSolutionSets, numUnreachedDestinationSets); for (List<Collection<Tree>> childSolutionSetCombination : childSolutionSetCombinations) { if (childSolutionSetCombination.size() < 2) continue; // only combine when we have more than on child solution for (List<Tree> solutionCombination : RheemCollections.streamedCrossProduct(childSolutionSetCombination)) { final Tree tree = ChannelConversionGraph.this.mergeTrees(solutionCombination); if (tree != null) {
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, SparkExecutor sparkExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length <= 1; assert outputs.length == this.getNumOutputs(); final Collection<Type> collection; if (this.collection != null) { collection = this.collection; } else { final CollectionChannel.Instance input = (CollectionChannel.Instance) inputs[0]; collection = input.provideCollection(); assert collection != null : String.format("Instance of %s is not providing a collection.", input.getChannel()); } final List<Type> list = RheemCollections.asList(collection); final RddChannel.Instance output = (RddChannel.Instance) outputs[0]; final JavaRDD<Type> rdd = sparkExecutor.sc.parallelize(list, sparkExecutor.getNumDefaultPartitions()); this.name(rdd); output.accept(rdd, sparkExecutor); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
/** * Validate that there is only a single element in the {@code collection} and return it. */ public static <T> T getSingle(Collection<T> collection) { Validate.isTrue(collection.size() == 1, "%s is not a singleton.", collection); return getAny(collection); }
/** * Creates the <i>power list</i> of the given {@code base} (akin to power sets of sets). */ public static <T> Collection<List<T>> createPowerList(Collection<T> base) { return createPowerList(base, base.size()); }
@Override public String toString() { return String.format("%s[%dx ~%s, %x]", this.getSimpleClassName(), this.alternatives.size(), RheemCollections.getAnyOptional(this.alternatives).orElse(null), this.hashCode()); }
/** * Helper method to create power lists. * * @param base the {@link List} whose power list is to be created * @param startIndex index of the first element in {@code base} to be considered * @param maxElements the maximum number of elements in {@link List}s within the power list * @param collector collects power list members */ private static <T> void createPowerListAux(List<T> base, int startIndex, int maxElements, List<List<T>> collector) { if (startIndex >= base.size()) { collector.add(Collections.emptyList()); } else { T head = base.get(startIndex); int collectorStartIndex = collector.size(); createPowerListAux(base, startIndex + 1, maxElements, collector); int collectorEndIndex = collector.size(); for (int i = collectorStartIndex; i < collectorEndIndex; i++) { final List<T> recursivelyCreatedElement = collector.get(i); if (recursivelyCreatedElement.size() < maxElements) { List<T> derivativeElement = new ArrayList<>(recursivelyCreatedElement.size() + 1); derivativeElement.add(head); derivativeElement.addAll(recursivelyCreatedElement); collector.add(derivativeElement); } } } }
public Junction(OutputSlot<?> sourceOutput, List<InputSlot<?>> targetInputs, List<OptimizationContext> optimizationContexts) { // Copy parameters. assert sourceOutput.getOwner().isExecutionOperator(); this.sourceOutput = sourceOutput; assert targetInputs.stream().allMatch(input -> input.getOwner().isExecutionOperator()); this.targetInputs = targetInputs; // Fill with nulls. this.targetChannels = RheemCollections.map(this.targetInputs, input -> null); // Get hold of an OptimizationContext. this.optimizationContexts = optimizationContexts; }
TaskActivator(ExecutionTask task, OptimizationContext.OperatorContext operatorContext, ExecutionState executionState) { assert operatorContext == null || task.getOperator() == operatorContext.getOperator() : String.format("Mismatch between %s and %s.", task, operatorContext); this.task = task; this.operatorContext = operatorContext; this.inputChannelInstances = RheemCollections.createNullFilledArrayList(this.getOperator().getNumInputs()); this.acceptFrom(executionState); }
/** * Retrieves the single(!) {@link Operator} that is <i>immediately</i> encased by this instance. * * @return the encased {@link Operator}s */ default Operator getContainedOperator() { return RheemCollections.getSingleOrNull(this.getContainedOperators()); }
/** * Validate that there is at most one element in the {@code collection} and return it (or {@code null} otherwise). */ public static <T> T getSingleOrNull(Collection<T> collection) { Validate.isTrue(collection.size() <= 1, "Expected 0 or 1 elements, found %d.", collection.size()); return collection.isEmpty() ? null : getAny(collection); }
@Override public void open(ExecutionContext ctx) { long numVertices = RheemCollections.getSingle(ctx.getBroadcast("numVertices")); this.initialRank = 1f / numVertices; }
/** * Find the {@link Operator} in this instance by their name. * * @param name the name of the {@link Operator}s * @return the matching {@link Operator} or {@code null} if none * @throws RheemException if there is more than one such {@link Operator} */ public Operator collectTopLevelOperatorByName(String name) throws RheemException{ return RheemCollections.getSingleOrNull(this.collectTopLevelOperatorsByName(name)); }
private StageExecution(ExecutionStage stage, OptimizationContext optimizationContext, ExecutionState executionState) { this.executionState = executionState; this.optimizationContext = optimizationContext; // Initialize the readyActivators. assert !stage.getStartTasks().isEmpty() : String.format("No start tasks for %s.", stage); stage.getStartTasks().forEach(this::scheduleStartTask); // Initialize the terminalTasks. this.terminalTasks = RheemCollections.asSet(stage.getTerminalTasks()); }