((CollectionChannel.Instance) inputs[0]).provideCollection().size();
public void accept(Collection<?> collection) { this.collection = collection; this.setMeasuredCardinality(this.collection.size()); }
ChannelInstance materializedInput, probingInput; if (inputs[0] instanceof CollectionChannel.Instance) { final Collection<InputType0> collection = ((CollectionChannel.Instance) inputs[0]).provideCollection(); final Stream<InputType1> stream = ((JavaChannelInstance) inputs[1]).provideStream(); output.<Tuple2<InputType0, InputType1>>accept( final Collection<InputType1> collection = ((CollectionChannel.Instance) inputs[1]).provideCollection(); output.<Tuple2<InputType0, InputType1>>accept( stream.flatMap(e0 -> collection.stream().map(
Stream<Input> stream1; if (inputs[0] instanceof CollectionChannel.Instance) { final Collection<Input> collection = ((CollectionChannel.Instance) inputs[0]).provideCollection(); stream0 = collection.stream(); stream1 = ((JavaChannelInstance) inputs[1]).provideStream(); } else if (inputs[1] instanceof CollectionChannel.Instance) { stream0 = ((JavaChannelInstance) inputs[0]).provideStream(); final Collection<Input> collection = ((CollectionChannel.Instance) inputs[1]).provideCollection(); stream1 = collection.stream(); } else {
final Collection<ConvergenceType> convergenceCollection = convergenceInput.provideCollection(); try { endloop = stoppingCondition.call(convergenceCollection); throw new RheemException(String.format("Could not evaluate stopping condition for %s.", this), e); executionLineageNode.addPredecessor(convergenceInput.getLineage()); break; default:
coGroups.add(new Tuple2<>(Collections.emptyList(), group1)); ((CollectionChannel.Instance) outputs[0]).accept(coGroups);
convergenceCollection = ((CollectionChannel.Instance) inputs[ITERATION_CONVERGENCE_INPUT_INDEX]).provideCollection();
final Collection<Input> collection = ((CollectionChannel.Instance) inputs[0]).provideCollection(); stream0 = collection.stream(); } else {
convergenceCollection = ((CollectionChannel.Instance) inputs[ITERATION_CONVERGENCE_INPUT_INDEX]).provideCollection(); executionLineageNode.addPredecessor(inputs[ITERATION_CONVERGENCE_INPUT_INDEX].getLineage());
assert inputs[CONVERGENCE_INPUT_INDEX] != null; convergenceCollection = ((CollectionChannel.Instance) inputs[CONVERGENCE_INPUT_INDEX]).provideCollection(); executionLineageNode.addPredecessor(inputs[CONVERGENCE_INPUT_INDEX].getLineage()); endloop = stoppingCondition.test(convergenceCollection);
@Override @SuppressWarnings("unchecked") public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); int sampleSize = this.getSampleSize(operatorContext); if (sampleSize >= datasetSize) { //return all ((CollectionChannel.Instance) outputs[0]).accept(((JavaChannelInstance) inputs[0]).provideStream().collect(Collectors.toList())); } else { long seed = this.getSeed(operatorContext); rand = new Random(seed); ((CollectionChannel.Instance) outputs[0]).accept(reservoirSample(rand, ((JavaChannelInstance) inputs[0]).<Type>provideStream().iterator(), sampleSize)); } return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
/** * Utility method to forward a {@link JavaChannelInstance} to another. * * @param input that should be forwarded * @param output to that should be forwarded */ static void forward(ChannelInstance input, ChannelInstance output) { // Do the forward. if (output instanceof CollectionChannel.Instance) { ((CollectionChannel.Instance) output).accept(((CollectionChannel.Instance) input).provideCollection()); } else if (output instanceof StreamChannel.Instance) { ((StreamChannel.Instance) output).accept(((JavaChannelInstance) input).provideStream()); } else { throw new RheemException(String.format("Cannot forward %s to %s.", input, output)); } // Manipulate the lineage. output.getLineage().addPredecessor(input.getLineage()); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, SparkExecutor sparkExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length <= 1; assert outputs.length == this.getNumOutputs(); final Collection<Type> collection; if (this.collection != null) { collection = this.collection; } else { final CollectionChannel.Instance input = (CollectionChannel.Instance) inputs[0]; collection = input.provideCollection(); assert collection != null : String.format("Instance of %s is not providing a collection.", input.getChannel()); } final List<Type> list = RheemCollections.asList(collection); final RddChannel.Instance output = (RddChannel.Instance) outputs[0]; final JavaRDD<Type> rdd = sparkExecutor.sc.parallelize(list, sparkExecutor.getNumDefaultPartitions()); this.name(rdd); output.accept(rdd, sparkExecutor); return ExecutionOperator.modelLazyExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, SparkExecutor sparkExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final RddChannel.Instance input = (RddChannel.Instance) inputs[0]; final CollectionChannel.Instance output = (CollectionChannel.Instance) outputs[0]; final Function2<Type, Type, Type> reduceFunction = sparkExecutor.getCompiler().compile(this.reduceDescriptor, this, operatorContext, inputs); final JavaRDD<Type> inputRdd = input.provideRdd(); List<Type> outputList = Collections.singletonList(inputRdd.reduce(reduceFunction)); output.accept(outputList); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final BinaryOperator<Type> reduceFunction = javaExecutor.getCompiler().compile(this.reduceDescriptor); JavaExecutor.openFunction(this, reduceFunction, inputs, operatorContext); final Optional<Type> reduction = ((JavaChannelInstance) inputs[0]).<Type>provideStream().reduce(reduceFunction); ((CollectionChannel.Instance) outputs[0]).accept(reduction.isPresent() ? Collections.singleton(reduction.get()) : Collections.emptyList()); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == 1; assert outputs.length == 1; CollectionChannel.Instance inputChannelInstance = (CollectionChannel.Instance) inputs[0]; final Collection<?> dataQuanta = inputChannelInstance.provideCollection(); Collection<Iterable<?>> dataQuantaGroup = new ArrayList<>(1); dataQuantaGroup.add(dataQuanta); CollectionChannel.Instance outputChannelInstance = (CollectionChannel.Instance) outputs[0]; outputChannelInstance.accept(dataQuantaGroup); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final Function<Type, KeyType> keyExtractor = javaExecutor.getCompiler().compile(this.keyDescriptor); final BinaryOperator<Type> reduceFunction = javaExecutor.getCompiler().compile(this.reduceDescriptor); JavaExecutor.openFunction(this, reduceFunction, inputs, operatorContext); final Map<KeyType, Type> reductionResult = ((JavaChannelInstance) inputs[0]).<Type>provideStream().collect( Collectors.groupingBy(keyExtractor, new ReducingCollector<>(reduceFunction)) ); ((CollectionChannel.Instance) outputs[0]).accept(reductionResult.values()); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }
@Override public Tuple<Collection<ExecutionLineageNode>, Collection<ChannelInstance>> evaluate( ChannelInstance[] inputs, ChannelInstance[] outputs, JavaExecutor javaExecutor, OptimizationContext.OperatorContext operatorContext) { assert inputs.length == this.getNumInputs(); assert outputs.length == this.getNumOutputs(); final JavaChannelInstance input = (JavaChannelInstance) inputs[0]; final long count; if (input instanceof CollectionChannel.Instance) { count = ((CollectionChannel.Instance) input).provideCollection().size(); } else { count = input.provideStream().count(); } ((CollectionChannel.Instance) outputs[0]).accept(Collections.singleton(count)); return ExecutionOperator.modelEagerExecution(inputs, outputs, operatorContext); }