public void setSerializersFrom(Integer from, Integer to) { StreamNode fromVertex = getStreamNode(from); StreamNode toVertex = getStreamNode(to); toVertex.setSerializerIn1(fromVertex.getTypeSerializerOut()); toVertex.setSerializerOut(fromVertex.getTypeSerializerIn1()); }
public void setSerializers(Integer vertexID, TypeSerializer<?> in1, TypeSerializer<?> in2, TypeSerializer<?> out) { StreamNode vertex = getStreamNode(vertexID); vertex.setSerializerIn1(in1); vertex.setSerializerIn2(in2); vertex.setSerializerOut(out); }
public void setOneInputStateKey(Integer vertexID, KeySelector<?, ?> keySelector, TypeSerializer<?> keySerializer) { StreamNode node = getStreamNode(vertexID); node.setStatePartitioner1(keySelector); node.setStateKeySerializer(keySerializer); }
public void setTwoInputStateKey(Integer vertexID, KeySelector<?, ?> keySelector1, KeySelector<?, ?> keySelector2, TypeSerializer<?> keySerializer) { StreamNode node = getStreamNode(vertexID); node.setStatePartitioner1(keySelector1); node.setStatePartitioner2(keySelector2); node.setStateKeySerializer(keySerializer); }
private ResourceSpec createChainedMinResources(Integer vertexID, List<StreamEdge> chainedOutputs) { ResourceSpec minResources = streamGraph.getStreamNode(vertexID).getMinResources(); for (StreamEdge chainable : chainedOutputs) { minResources = minResources.merge(chainedMinResources.get(chainable.getTargetId())); } return minResources; }
private void head(StreamOperator<?> headOperator, OperatorID headOperatorID) { headConfig.setStreamOperator(headOperator); headConfig.setOperatorID(headOperatorID); headConfig.setChainStart(); headConfig.setChainIndex(chainIndex); }
private ResourceSpec createChainedPreferredResources(Integer vertexID, List<StreamEdge> chainedOutputs) { ResourceSpec preferredResources = streamGraph.getStreamNode(vertexID).getPreferredResources(); for (StreamEdge chainable : chainedOutputs) { preferredResources = preferredResources.merge(chainedPreferredResources.get(chainable.getTargetId())); } return preferredResources; }
public void setParallelism(Integer vertexID, int parallelism) { if (getStreamNode(vertexID) != null) { getStreamNode(vertexID).setParallelism(parallelism); } }
public void setResources(int vertexID, ResourceSpec minResources, ResourceSpec preferredResources) { if (getStreamNode(vertexID) != null) { getStreamNode(vertexID).setResources(minResources, preferredResources); } }
public void setMaxParallelism(int vertexID, int maxParallelism) { if (getStreamNode(vertexID) != null) { getStreamNode(vertexID).setMaxParallelism(maxParallelism); } }
public void setBufferTimeout(Integer vertexID, long bufferTimeout) { if (getStreamNode(vertexID) != null) { getStreamNode(vertexID).setBufferTimeout(bufferTimeout); } }
public <IN, OUT> void setOperator(Integer vertexID, StreamOperator<OUT> operatorObject) { getStreamNode(vertexID).setOperator(operatorObject); }
public void setInputFormat(Integer vertexID, InputFormat<?, ?> inputFormat) { getStreamNode(vertexID).setInputFormat(inputFormat); }
private static boolean isCustomPartitioned(List<StreamEdge> edges) { boolean result = true; for (StreamEdge edge: edges) { if (!(edge.getPartitioner() instanceof CustomPartitionerWrapper)) { result = false; } } return result; }
/** * Sets up task chains from the source {@link StreamNode} instances. * * <p>This will recursively create all {@link JobVertex} instances. */ private void setChaining(Map<Integer, byte[]> hashes, List<Map<Integer, byte[]>> legacyHashes, Map<Integer, List<Tuple2<byte[], byte[]>>> chainedOperatorHashes) { for (Integer sourceNodeId : streamGraph.getSourceIDs()) { createChain(sourceNodeId, sourceNodeId, hashes, legacyHashes, 0, chainedOperatorHashes); } }
/** * Returns the checkpointing mode. * * @return checkpointing mode */ public CheckpointingMode getCheckpointMode() { return streamConfig.getCheckpointMode(); }
/** * Returns true if checkpointing is enabled for the running job. * * @return true if checkpointing is enabled. */ public boolean isCheckpointingEnabled() { return streamConfig.isCheckpointingEnabled(); }
/** * Returns the buffer timeout of the job. * * @return buffer timeout (in milliseconds) */ public long getBufferTimeout() { return streamConfig.getBufferTimeout(); }
/** * This starts the actual transformation, beginning from the sinks. */ private StreamGraph generateInternal(List<StreamTransformation<?>> transformations) { for (StreamTransformation<?> transformation: transformations) { transform(transformation); } return streamGraph; }
private static boolean isPartitioned(List<StreamEdge> edges) { boolean result = true; for (StreamEdge edge: edges) { if (!(edge.getPartitioner() instanceof KeyGroupStreamPartitioner)) { result = false; } } return result; }