public DeltaIterationBase(BinaryOperatorInformation<ST, WT, ST> operatorInfo, int[] keyPositions, String name) { super(new UserCodeClassWrapper<AbstractRichFunction>(AbstractRichFunction.class), operatorInfo, name); this.solutionSetKeyFields = keyPositions; solutionSetPlaceholder = new SolutionSetPlaceHolder<ST>(this, new OperatorInformation<ST>(operatorInfo.getFirstInputType())); worksetPlaceholder = new WorksetPlaceHolder<WT>(this, new OperatorInformation<WT>(operatorInfo.getSecondInputType())); }
@Override public UnaryOperatorInformation<OUT, OUT> getOperatorInfo() { TypeInformation<OUT> previousOut = input.getOperatorInfo().getOutputType(); return new UnaryOperatorInformation<OUT, OUT>(previousOut, previousOut); }
public Union(Operator<T> input1, Operator<T> input2, String unionLocationName) { this(new BinaryOperatorInformation<T, T, T>(input1.getOperatorInfo().getOutputType(), input1.getOperatorInfo().getOutputType(), input1.getOperatorInfo().getOutputType()), unionLocationName); setFirstInput(input1); setSecondInput(input2); }
protected GenericDataSourceBase<OUT, ?> translateToDataFlow() { String name = this.name != null ? this.name : "at " + dataSourceLocationName + " (" + inputFormat.getClass().getName() + ")"; if (name.length() > 150) { name = name.substring(0, 150); } @SuppressWarnings({"unchecked", "rawtypes"}) GenericDataSourceBase<OUT, ?> source = new GenericDataSourceBase(this.inputFormat, new OperatorInformation<OUT>(getType()), name); source.setParallelism(parallelism); if (this.parameters != null) { source.getParameters().addAll(this.parameters); } if (this.splitDataProperties != null) { source.setSplitDataProperties(this.splitDataProperties); } return source; }
@SuppressWarnings("unchecked") private static <T> TypeInformation<T> getTypeInfoFromSource(SourcePlanNode node) { Operator<?> op = node.getOptimizerNode().getOperator(); if (op instanceof GenericDataSourceBase) { return ((GenericDataSourceBase<T, ?>) op).getOperatorInfo().getOutputType(); } else { throw new RuntimeException("Wrong operator type found in post pass."); } }
private static final DataSourceNode getSourceNode() { return new DataSourceNode(new GenericDataSourceBase<String, TextInputFormat>( new TextInputFormat(new Path("/ignored")), new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO), "source")); } }
@Override protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) { TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType(); int[] sortColumns = this.partitionOrdering.getFieldPositions(); boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections(); final TypeComparator<IN> sortComparator; if (inputType instanceof CompositeType) { sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig); } else if (inputType instanceof AtomicType) { sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig); } else { throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet."); } Collections.sort(inputData, new Comparator<IN>() { @Override public int compare(IN o1, IN o2) { return sortComparator.compare(o1, o2); } }); return inputData; } }
private static DataSourceNode getSourceNode() { return new DataSourceNode(new GenericDataSourceBase<String, TextInputFormat>(new TextInputFormat(new Path("/")), new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO))); }
type = input1.getOperatorInfo().getOutputType(); } else if (input2.length > 0 && input2[0] != null) { type = input2[0].getOperatorInfo().getOutputType(); } else { throw new IllegalArgumentException("Could not determine type information from inputs.");
@Test public void testDataSourcePlain() { try { TestNonRichInputFormat in = new TestNonRichInputFormat(); GenericDataSourceBase<String, TestNonRichInputFormat> source = new GenericDataSourceBase<String, TestNonRichInputFormat>( in, new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO), "testSource"); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.disableObjectReuse(); List<String> resultMutableSafe = source.executeOnCollections(null, executionConfig); in.reset(); executionConfig.enableObjectReuse(); List<String> resultRegular = source.executeOnCollections(null, executionConfig); assertEquals(asList(TestIOData.NAMES), resultMutableSafe); assertEquals(asList(TestIOData.NAMES), resultRegular); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) { PlanNode source = channel.getSource(); Operator<?> javaOp = source.getProgramOperator(); TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType(); return createComparator(type, channel.getShipStrategyKeys(), getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig); }
GenericDataSourceBase<String, TestRichInputFormat> source = new GenericDataSourceBase<String, TestRichInputFormat>( in, new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO), "testSource");
TypeSerializer<OUT> serializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);
public DeltaIterationBase(BinaryOperatorInformation<ST, WT, ST> operatorInfo, int[] keyPositions, String name) { super(new UserCodeClassWrapper<AbstractRichFunction>(AbstractRichFunction.class), operatorInfo, name); this.solutionSetKeyFields = keyPositions; solutionSetPlaceholder = new SolutionSetPlaceHolder<ST>(this, new OperatorInformation<ST>(operatorInfo.getFirstInputType())); worksetPlaceholder = new WorksetPlaceHolder<WT>(this, new OperatorInformation<WT>(operatorInfo.getSecondInputType())); }
/** * @param criterion */ public <X> void setTerminationCriterion(Operator<X> criterion) { TypeInformation<X> type = criterion.getOperatorInfo().getOutputType(); FlatMapOperatorBase<X, X, TerminationCriterionMapper<X>> mapper = new FlatMapOperatorBase<X, X, TerminationCriterionMapper<X>>( new TerminationCriterionMapper<X>(), new UnaryOperatorInformation<X, X>(type, type), "Termination Criterion Aggregation Wrapper"); mapper.setInput(criterion); this.terminationCriterion = mapper; this.getAggregators().registerAggregationConvergenceCriterion(TERMINATION_CRITERION_AGGREGATOR_NAME, new TerminationCriterionAggregator(), new TerminationCriterionAggregationConvergence()); }
public DeltaIterationBase(BinaryOperatorInformation<ST, WT, ST> operatorInfo, int[] keyPositions, String name) { super(new UserCodeClassWrapper<AbstractRichFunction>(AbstractRichFunction.class), operatorInfo, name); this.solutionSetKeyFields = keyPositions; solutionSetPlaceholder = new SolutionSetPlaceHolder<ST>(this, new OperatorInformation<ST>(operatorInfo.getFirstInputType())); worksetPlaceholder = new WorksetPlaceHolder<WT>(this, new OperatorInformation<WT>(operatorInfo.getSecondInputType())); }
@SuppressWarnings("unchecked") protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception { OutputFormat<IN> format = this.formatWrapper.getUserCodeObject(); TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();
protected GenericDataSourceBase<OUT, ?> translateToDataFlow() { String name = this.name != null ? this.name : "at " + dataSourceLocationName + " (" + inputFormat.getClass().getName() + ")"; if (name.length() > 150) { name = name.substring(0, 150); } @SuppressWarnings({"unchecked", "rawtypes"}) GenericDataSourceBase<OUT, ?> source = new GenericDataSourceBase(this.inputFormat, new OperatorInformation<OUT>(getType()), name); source.setParallelism(parallelism); if (this.parameters != null) { source.getParameters().addAll(this.parameters); } if (this.splitDataProperties != null) { source.setSplitDataProperties(this.splitDataProperties); } return source; }
TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType(); (source.getDriverStrategy() == DriverStrategy.SORTED_GROUP_COMBINE || source.getDriverStrategy() == DriverStrategy.ALL_GROUP_REDUCE_COMBINE)) { GroupReduceOperatorBase<?, ?, ?> groupNode = (GroupReduceOperatorBase<?, ?, ?>) javaOp; type = groupNode.getInput().getOperatorInfo().getOutputType(); type = groupNode.getInput().getOperatorInfo().getOutputType();
protected GenericDataSourceBase<OUT, ?> translateToDataFlow() { String name = this.name != null ? this.name : "at " + dataSourceLocationName + " (" + inputFormat.getClass().getName() + ")"; if (name.length() > 150) { name = name.substring(0, 150); } @SuppressWarnings({"unchecked", "rawtypes"}) GenericDataSourceBase<OUT, ?> source = new GenericDataSourceBase(this.inputFormat, new OperatorInformation<OUT>(getType()), name); source.setParallelism(parallelism); if (this.parameters != null) { source.getParameters().addAll(this.parameters); } if (this.splitDataProperties != null) { source.setSplitDataProperties(this.splitDataProperties); } return source; }