SourcePlanNode candidate = new SourcePlanNode(this, "DataSource ("+this.getOperator().getName()+")", this.gprops, this.lprops); candidate.updatePropertiesWithUniqueSets(getUniqueFields()); estimator.addFileInputCost(this.estimatedOutputSize, costs); candidate.setCosts(costs); } else { estimator.addFileInputCost(this.estimatedOutputSize * this.getParallelism(), costs); candidate.setCosts(costs);
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
@SuppressWarnings("unchecked") private static <T> TypeInformation<T> getTypeInfoFromSource(SourcePlanNode node) { Operator<?> op = node.getOptimizerNode().getOperator(); if (op instanceof GenericDataSourceBase) { return ((GenericDataSourceBase<T, ?>) op).getOperatorInfo().getOutputType(); } else { throw new RuntimeException("Wrong operator type found in post pass."); } }
@Test public void testNoPartialSolutionFoundSingleInputOnly() { try { SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source"); SourcePlanNode otherTarget = new SourcePlanNode(getSourceNode(), "Source");
((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
public SourcePlanNode(DataSourceNode template, String nodeName, GlobalProperties gprops, LocalProperties lprops) { super(template, nodeName, DriverStrategy.NONE); this.globalProps = gprops; this.localProps = lprops; updatePropertiesWithUniqueSets(template.getUniqueFields()); }
@Test public void testNoPartialSolutionFoundTwoInputOperator() { try { SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution"); SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1"); SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
public SourcePlanNode(DataSourceNode template, String nodeName, GlobalProperties gprops, LocalProperties lprops) { super(template, nodeName, DriverStrategy.NONE); this.globalProps = gprops; this.localProps = lprops; updatePropertiesWithUniqueSets(template.getUniqueFields()); }
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
SourcePlanNode candidate = new SourcePlanNode(this, "DataSource ("+this.getOperator().getName()+")", this.gprops, this.lprops); candidate.updatePropertiesWithUniqueSets(getUniqueFields()); estimator.addFileInputCost(this.estimatedOutputSize, costs); candidate.setCosts(costs); } else { estimator.addFileInputCost(this.estimatedOutputSize * this.getParallelism(), costs); candidate.setCosts(costs);
@Test public void testGetEstimatesNoReplicationFactor() { final long NUM_RECORD = 1001; final long SIZE = 467131; DataSourceNode source = getSourceNode(); SourcePlanNode planNode = new SourcePlanNode(source, "test node"); Channel channel = new Channel(planNode); // no estimates here Assert.assertEquals(-1, channel.getEstimatedOutputSize()); Assert.assertEquals(-1, channel.getEstimatedNumRecords()); // set estimates source.setEstimatedNumRecords(NUM_RECORD); source.setEstimatedOutputSize(SIZE); Assert.assertEquals(SIZE, channel.getEstimatedOutputSize()); Assert.assertEquals(NUM_RECORD, channel.getEstimatedNumRecords()); }
((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
@SuppressWarnings("unchecked") private static <T> TypeInformation<T> getTypeInfoFromSource(SourcePlanNode node) { Operator<?> op = node.getOptimizerNode().getOperator(); if (op instanceof GenericDataSourceBase) { return ((GenericDataSourceBase<T, ?>) op).getOperatorInfo().getOutputType(); } else { throw new RuntimeException("Wrong operator type found in post pass."); } }
public SourcePlanNode(DataSourceNode template, String nodeName, GlobalProperties gprops, LocalProperties lprops) { super(template, nodeName, DriverStrategy.NONE); this.globalProps = gprops; this.localProps = lprops; updatePropertiesWithUniqueSets(template.getUniqueFields()); }
private InputFormatVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException { final InputFormatVertex vertex = new InputFormatVertex(node.getNodeName()); final TaskConfig config = new TaskConfig(vertex.getConfiguration()); vertex.setResources(node.getMinResources(), node.getPreferredResources()); vertex.setInvokableClass(DataSourceTask.class); vertex.setFormatDescription(getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper())); // set user code config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper()); config.setStubParameters(node.getProgramOperator().getParameters()); config.setOutputSerializer(node.getSerializer()); return vertex; }
SourcePlanNode candidate = new SourcePlanNode(this, "DataSource ("+this.getOperator().getName()+")", this.gprops, this.lprops); candidate.updatePropertiesWithUniqueSets(getUniqueFields()); estimator.addFileInputCost(this.estimatedOutputSize, costs); candidate.setCosts(costs); } else { estimator.addFileInputCost(this.estimatedOutputSize * this.getParallelism(), costs); candidate.setCosts(costs);
@Test public void testGetEstimatesWithReplicationFactor() { final long NUM_RECORD = 1001; final long SIZE = 467131; final int REPLICATION = 23; DataSourceNode source = getSourceNode(); SourcePlanNode planNode = new SourcePlanNode(source, "test node"); Channel channel = new Channel(planNode); channel.setReplicationFactor(REPLICATION); // no estimates here Assert.assertEquals(-1, channel.getEstimatedOutputSize()); Assert.assertEquals(-1, channel.getEstimatedNumRecords()); // set estimates source.setEstimatedNumRecords(NUM_RECORD); source.setEstimatedOutputSize(SIZE); Assert.assertEquals(SIZE * REPLICATION, channel.getEstimatedOutputSize()); Assert.assertEquals(NUM_RECORD * REPLICATION, channel.getEstimatedNumRecords()); }
((SourcePlanNode) node).setSerializer(createSerializer(typeInfo));
@SuppressWarnings("unchecked") private static <T> TypeInformation<T> getTypeInfoFromSource(SourcePlanNode node) { Operator<?> op = node.getOptimizerNode().getOperator(); if (op instanceof GenericDataSourceBase) { return ((GenericDataSourceBase<T, ?>) op).getOperatorInfo().getOutputType(); } else { throw new RuntimeException("Wrong operator type found in post pass."); } }